def test_parse_with_invalid_csv(): # Since we accept text columns, we presume that strings appear to be errors # only if it's less than 10% of them in a column. Hence it can be tested # only on a dataset with minimum 11 columns, where one contains string. # data = '1 2 3 4\n1 2 3 4\n5 6 a 8\n7 6 5 2\n8 8 8 8\n2 2 3 9\n5 6 7 8\n12 13 45 56\n12 43 6 7\n9 9 9 0\n1 2 5 0\n' log = """ Parsing CSV with whitespace (tab) as delimiter. Found 4 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... No header found, first row contains data. Found 1 row with invalid values: - row 3, column 3 Found 10 samples. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_notify_data_invalid(): data = gen_data() log = """ Image dataset unpacked. Parsing... This file doesn't contain a supported data format.""" notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with tempdir() as td: td.child('class1').mkdir() td.child('class2').mkdir() td.child('class3').mkdir() td.child('class4').mkdir() files = [ td.child('class3', 'test.txt'), td.child('1.jpg'), ] for f in (files): f = open(f, 'w') f.write(data) f.close() with TempFile(suffix='.tar.bz2') as fname: with tarfile.open(fname, 'w:bz2') as z: with cwd(td): for f in (files): f = f.replace(td, './') z.add(f) with global_notify(notify): with pytest.raises(InvalidDataFile): parse_archive(Archive(fname)) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def fail_template(data,log): notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.ts') as ts: with global_notify(notify): with pytest.raises(InvalidTimeseries): parse(ts) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def pass_template(data,log,exp_meta): notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.ts') as ts: with global_notify(notify): meta = parse(ts) for key in exp_meta: assert meta[key] == exp_meta[key] rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_notify_archive_invalid(): data = 'thequickbrownfoxjumpsoverthelazydog' log = 'Unknown file format.' notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.foo') as foo: with global_notify(notify): with pytest.raises(InvalidArchive): parse(foo) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_no_data(): data = '' log = """ First row is empty, it must contain headers or data. This means your file isn't properly formatted (or you submitted another type of file). """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): with pytest.raises(InvalidCSV): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_with_invalid_csv_other_file_type(): data = '%PDF-1.4\n' log = """ CSV doesn't contain a valid delimiter. This means your file isn't properly formatted (or you submitted another type of file). """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): with pytest.raises(InvalidCSV): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_notify_archive_csv_valid(): data = gen_data() log = """ Image dataset unpacked. Parsing... CSV file .//1.csv unpacked. Parsing CSV with whitespace (tab) as delimiter. Found 3 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... The dataset appears to have a header. Found 2 samples.""" notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with tempdir() as td: td.child('class1').mkdir() td.child('class2').mkdir() td.child('class3').mkdir() td.child('class4').mkdir() files = [ td.child('class1', 'f1.jpg'), td.child('class1', 'f2.JPG'), td.child('class2', 'f1.jpg'), td.child('class2', 'f2.bMp'), td.child('class2', 'f4jpg.Jpeg'), td.child('class3', 'test.txt'), td.child('1.csv'), ] for f in (files): f = open(f, 'w') f.write(data) f.close() with open(td.child('1.csv'), 'w') as f: f.write('one two free\r1 2 3\r4 5 6') with TempFile(suffix='.tar.bz2') as fname: with tarfile.open(fname, 'w:bz2') as z: with cwd(td): for f in (files): f = f.replace(td, './') z.add(f) with global_notify(notify): parse_archive(Archive(fname)) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_not_enough_columns(): data = '3,\n 4,,5\n6,8,9\n' log = """ Parsing CSV with comma as delimiter. With selected delimiter found only 1 columns in first row, must be at least 2. This means your file isn't properly formatted (or you submitted another type of file). """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): with pytest.raises(InvalidCSV): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_with_invalid_csv_other_file_type_valid_delimiter(): data = '%PDF,-1.4\nadsadsadsad' log = """ Parsing CSV with comma as delimiter. Found 2 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... The dataset is empty or isn't properly formatted. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): with pytest.raises(InvalidCSV): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
def test_notify_archive_image_skipped(): data = gen_data() log = """ Image dataset unpacked. Parsing... 8 images found. Skipped 3 images with leading dot or without class. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with tempdir() as td: td.child('class1').mkdir() td.child('class2').mkdir() td.child('class2').child('.class22').mkdir() td.child('class3').mkdir() td.child('class4').mkdir() files = [ td.child('class1', 'f1.jpg'), td.child('class1', 'f2.JPG'), td.child('class2', 'f1.jpg'), td.child('class2', '.f1.jpg'), td.child('class2', 'f2.bMp'), td.child('class2', '.class22', 'ff2.jpg'), td.child('class2', 'f4jpg.Jpeg'), td.child('class3', 'test.txt'), td.child('f1.jpeg'), ] for f in (files): f = open(f, 'w') f.write(data) f.close() with TempFile(suffix='.zip') as fname: with zipfile.ZipFile(fname, 'w', compression=zipfile.ZIP_DEFLATED) as z: with cwd(td): for f in (files): f = f.replace(td, './') z.write(f) with global_notify(notify): parse_archive(Archive(fname)) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_bad_column_data_first_row(): data = '3,\\x00,4,3\n4,\n6,8,9\n' log = """ Parsing CSV with comma as delimiter. Found 4 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... The dataset is empty or isn't properly formatted. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): with pytest.raises(InvalidCSV): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_different_delimiters_per_Row(): data = '4 5\n4,6\n' log = """ Parsing CSV with whitespace (tab) as delimiter. Found 2 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... The dataset is empty or isn't properly formatted. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): with pytest.raises(InvalidCSV): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_with_csv(): data = 'a,b,c,d\n\n1,2,3,4\n5,6,7,8\n' log = """ Parsing CSV with comma as delimiter. Found 4 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... The dataset appears to have a header. Found 2 samples. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): meta = parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert meta['data_type'] == 'GENERAL' assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_with_invalid_csv_with_null_bytes(): data = "1 2 3 4\n1 2 3 4\n5 6 \\x00 8\n7 6 5 2\n8 8 8 8\n2 2 3 9\n5 6 7 8\n12 13 45 56\n12 43 6 7\n9 9 9 0\n1 2 5 0\n" log = """ Parsing CSV with whitespace (tab) as delimiter. Found 4 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... No header found, first row contains data. Found 1 row with invalid values: - row 3, column 3 Found 10 samples. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_notify_archive_ts_valid(): data = gen_data() log = """ Image dataset unpacked. Parsing... Timeseries data .//class3/test.ts unpacked. Parsing... First timestep has 3 inputs and 2 outputs. Applying this requirement to the entire file.""" notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with tempdir() as td: td.child('class1').mkdir() td.child('class2').mkdir() td.child('class3').mkdir() td.child('class4').mkdir() files = [ td.child('class1', 'f1.jpg'), td.child('class1', 'f2.JPG'), td.child('class2', 'f1.jpg'), td.child('class2', 'f2.bMp'), td.child('class2', 'f4jpg.Jpeg'), td.child('class3', 'test.ts'), td.child('f1.jpeg'), ] for f in (files): f = open(f, 'w') f.write(data) f.close() with open(td.child('class3', 'test.ts'), 'w') as f: f.write('1,2,3|0,1; 2.3,4,1|0,1; 1.1, 0., 0.0|1,0\n\n2,2,2|0,1;') with TempFile(suffix='.tar.gz') as fname: with tarfile.open(fname, 'w:gz') as z: with cwd(td): for f in (files): f = f.replace(td, './') z.add(f) with global_notify(notify): parse_archive(Archive(fname)) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called
def test_parse_rows_non_consistent(): # Warning! No error here: inconsisten rows ignored by np.genfromtxt data = '1,2,3,4,5,6,7,8\n8,7,6,5,4,3,2,1\n56,34,34\n' log = """ Parsing CSV with comma as delimiter. Found 8 fields in first row, assume all the rows have this number of fields. Parsing... Analyzing data... No header found, first row contains data. Found 1 row with invalid values: - row 3, column 4 Found 2 samples. """ notify = mock.MagicMock() notify.send = mock.MagicMock() notify.admin_send = mock.MagicMock() with TempFile(data, suffix='.csv') as csv: with global_notify(notify): meta = parse(csv) rval = '\n'.join(x[0][0] for x in notify.send.call_args_list) assert meta['data_type'] == 'GENERAL' assert rval == '\n'.join(x.strip() for x in log.strip().split('\n')) assert not notify.admin_send.called