def test_parse_with_invalid_csv():
# Since we accept text columns, we presume that strings appear to be errors
# only if it's less than 10% of them in a column. Hence it can be tested
# only on a dataset with minimum 11 columns, where one contains string.
#
data = '1 2 3 4\n1 2 3 4\n5 6 a 8\n7 6 5 2\n8 8 8 8\n2 2 3 9\n5 6 7 8\n12 13 45 56\n12 43 6 7\n9 9 9 0\n1 2 5 0\n'
log = """
Parsing CSV with whitespace (tab) as delimiter.
Found 4 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
No header found, first row contains data.
Found 1 row with invalid values:
- row 3, column 3
Found 10 samples.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def fail_template(data,log):
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.ts') as ts:
with global_notify(notify):
with pytest.raises(InvalidTimeseries):
parse(ts)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_notify_archive_invalid():
data = 'thequickbrownfoxjumpsoverthelazydog'
log = 'Unknown file format.'
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.foo') as foo:
with global_notify(notify):
with pytest.raises(InvalidArchive):
parse(foo)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_with_archive():
data = gen_data()
with tempdir() as td:
td.child('class1').mkdir()
td.child('class2').mkdir()
td.child('class3').mkdir()
td.child('class4').mkdir()
files = [
td.child('class1', 'f1.jpg'),
td.child('class1', 'f2.JPG'),
td.child('class2', 'f1.jpg'),
td.child('class2', 'f2.bMp'),
td.child('class2', 'f4jpg.Jpeg'),
td.child('class3', 'test.txt'),
td.child('f1.jpeg'),
]
for f in (files):
f = open(f, 'w')
f.write(data)
f.close()
with TempFile(suffix='.zip') as fname:
with zipfile.ZipFile(fname, 'w',
compression=zipfile.ZIP_DEFLATED) as z:
with cwd(td):
for f in (files):
f = f.replace(td, './')
z.write(f)
meta = parse(fname)
assert meta == {
'data_type': 'IMAGES',
'classes': {
'class1': 2,
'class2': 3
}
}
def test_parse_no_data():
data = ''
log = """
First row is empty, it must contain headers or data.
This means your file isn't properly formatted
(or you submitted another type of file).
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
with pytest.raises(InvalidCSV):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_with_invalid_csv_other_file_type():
data = '%PDF-1.4\n'
log = """
CSV doesn't contain a valid delimiter.
This means your file isn't properly formatted
(or you submitted another type of file).
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
with pytest.raises(InvalidCSV):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_with_invalid_csv_other_file_type_valid_delimiter():
data = '%PDF,-1.4\nadsadsadsad'
log = """
Parsing CSV with comma as delimiter.
Found 2 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
The dataset is empty or isn't properly formatted.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
with pytest.raises(InvalidCSV):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
def test_parse_not_enough_columns():
data = '3,\n 4,,5\n6,8,9\n'
log = """
Parsing CSV with comma as delimiter.
With selected delimiter found only 1 columns in first row, must be at least 2.
This means your file isn't properly formatted
(or you submitted another type of file).
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
with pytest.raises(InvalidCSV):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_different_delimiters_per_Row():
data = '4 5\n4,6\n'
log = """
Parsing CSV with whitespace (tab) as delimiter.
Found 2 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
The dataset is empty or isn't properly formatted.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
with pytest.raises(InvalidCSV):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_bad_column_data_first_row():
data = '3,\\x00,4,3\n4,\n6,8,9\n'
log = """
Parsing CSV with comma as delimiter.
Found 4 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
The dataset is empty or isn't properly formatted.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
with pytest.raises(InvalidCSV):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_with_invalid_csv_with_null_bytes():
data = "1 2 3 4\n1 2 3 4\n5 6 \\x00 8\n7 6 5 2\n8 8 8 8\n2 2 3 9\n5 6 7 8\n12 13 45 56\n12 43 6 7\n9 9 9 0\n1 2 5 0\n"
log = """
Parsing CSV with whitespace (tab) as delimiter.
Found 4 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
No header found, first row contains data.
Found 1 row with invalid values:
- row 3, column 3
Found 10 samples.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def pass_template(data,log,exp_meta):
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.ts') as ts:
with global_notify(notify):
meta = parse(ts)
for key in exp_meta:
assert meta[key] == exp_meta[key]
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_with_ts():
with TempFile('1,2,3|1, 0;2,3,4|1,0', suffix='.ts') as ts:
meta = parse(ts)
assert meta == {
'data_type': 'TIMESERIES',
'data_rows': 1,
'empty_rows': 0,
'min_timesteps': 2,
'max_timesteps': 2,
'classes': {
'0': 2
},
'binary_input': False,
'binary_output': True,
'input_size': 3,
'output_size': 2
}
def test_parse_with_csv():
data = 'a,b,c,d\n\n1,2,3,4\n5,6,7,8\n'
log = """
Parsing CSV with comma as delimiter.
Found 4 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
The dataset appears to have a header.
Found 2 samples.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
meta = parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert meta['data_type'] == 'GENERAL'
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called
def test_parse_with_csv_zip():
with TempFile(suffix='.zip') as t:
with zipfile.ZipFile(t, 'w') as z:
z.writestr('test.csv', '1 2 3\n2 3 4')
meta = parse(t)
assert meta == {
'data_type': 'GENERAL',
'version': 3,
'size': 12,
'data_rows': 2,
'empty_rows': 0,
'invalid_rows': 0,
'num_columns': 3,
'delimeter': '\s+',
'with_header': False,
'archive_path': 'test.csv',
'last_column_info': {
'classes': {
'3': 1,
'4': 1
},
'distrib': {
'3': 0.5,
'4': 0.5
},
'max': 4.,
'min': 3.,
'unique': 2,
},
'histogram': [[1, 1], [1, 1], [1, 1]],
'bins': [[1.0, 1.5, 2.0], [2.0, 2.5, 3.0], [3.0, 3.5, 4.0]],
'uniques_per_col': [2, 2, 2],
'classes': [[], [], []],
'dtypes': ['i', 'i', 'i'],
'locked': [False, False, False],
'names': ['1', '2', '3'],
'mean': [1.5, 2.5, 3.5],
'stdev': [0.707107, 0.707107, 0.707107],
'max': [2, 3, 4],
'min': [1, 2, 3]
}
def test_parse_with_csv():
with TempFile('1,2,3\r\n2,3,4', suffix='.csv') as csv:
meta = parse(csv)
assert meta == {
'data_type': 'GENERAL',
'version': 3,
'size': 12,
'data_rows': 2,
'empty_rows': 0,
'invalid_rows': 0,
'num_columns': 3,
'delimeter': '\s*,\s*',
'with_header': False,
'last_column_info': {
'classes': {
'3': 1,
'4': 1
},
'distrib': {
'3': 0.5,
'4': 0.5
},
'max': 4.,
'min': 3.,
'unique': 2,
},
'histogram': [[1, 1], [1, 1], [1, 1]],
'bins': [[1.0, 1.5, 2.0], [2.0, 2.5, 3.0], [3.0, 3.5, 4.0]],
'uniques_per_col': [2, 2, 2],
'classes': [[], [], []],
'dtypes': ['i', 'i', 'i'],
'locked': [False, False, False],
'names': ['1', '2', '3'],
'mean': [1.5, 2.5, 3.5],
'stdev': [0.707107, 0.707107, 0.707107],
'max': [2, 3, 4],
'min': [1, 2, 3]
}
def test_parse_rows_non_consistent():
# Warning! No error here: inconsisten rows ignored by np.genfromtxt
data = '1,2,3,4,5,6,7,8\n8,7,6,5,4,3,2,1\n56,34,34\n'
log = """
Parsing CSV with comma as delimiter.
Found 8 fields in first row, assume all the rows have this number of fields.
Parsing...
Analyzing data...
No header found, first row contains data.
Found 1 row with invalid values:
- row 3, column 4
Found 2 samples.
"""
notify = mock.MagicMock()
notify.send = mock.MagicMock()
notify.admin_send = mock.MagicMock()
with TempFile(data, suffix='.csv') as csv:
with global_notify(notify):
meta = parse(csv)
rval = '\n'.join(x[0][0] for x in notify.send.call_args_list)
assert meta['data_type'] == 'GENERAL'
assert rval == '\n'.join(x.strip() for x in log.strip().split('\n'))
assert not notify.admin_send.called