def test_ts():
clean_working_dir()
key = 'fixtures/dmworker/manual.ts'
df = S3File(key)
assert not df.is_compressed
local_name = get_local_file_name(df.key)
df.download(local_name)
assert local_name.exists()
assert md5(local_name) == df.etag()
def teardown_module(module):
clean_working_dir()
key = 'fixtures/dmworker/iris.csv.zip'
key = get_key(key)
if key is not None:
key.delete()
key = 'fixtures/dmworker/invalid.csv.zip'
key = get_key(key)
if key is not None:
key.delete()
def setup_module(module):
clean_working_dir()
key = 'fixtures/dmworker/iris.csv.zip'
key = get_key(key)
if key is not None:
key.delete()
key = 'fixtures/dmworker/manual.ts.zip'
key = get_key(key)
if key is not None:
key.delete()
def test_swap_to_zip():
clean_working_dir()
key = 'fixtures/dmworker/manual.ts.zip'
key = get_key(key)
if key is not None:
key.delete()
key = 'fixtures/dmworker/manual.ts'
df = S3File(key)
assert not df.is_compressed
df.swap_to_zip()
assert df._key is None
def test_csv_text_labels():
clean_working_dir()
key = 'fixtures/dmworker/invalid.csv'
meta = parser.run(key, notify)
assert meta == {
'size':
85,
'data_rows':
3,
'uniques_per_col': [3, 3, 1, 3],
'key':
'fixtures/dmworker/invalid.csv.zip',
'data_type':
'GENERAL',
'invalid_rows':
0,
'version':
3,
'histogram': [[1, 1, 1], [1, 1, 1], [3], [1, 1, 1]],
'bins': [[0, 0.666667, 1.33333, 2], [0.1, 0.333333, 0.566667, 0.8],
[0.11, 0.11], [0, 0.666667, 1.33333, 2]],
'dtypes': ['S', 'f', 'f', 'S'],
'classes': [['A', 'M', 'U'], [], [], ['class1', 'class2', 'class3']],
'last_column_info': {
'distrib': {
'class2': 0.333333,
'class3': 0.333333,
'class1': 0.333333
},
'classes': {
'class2': 1,
'class3': 1,
'class1': 1
},
'unique': 3,
'max': 2,
'min': 0
},
'names': ['column1', 'column2', 'column3', 'column4'],
'delimeter':
'\\s+',
'num_columns':
4,
'locked': [True, True, True, True],
'with_header':
True,
'empty_rows':
0,
'mean': [None, 0.466667, 0.11, None],
'stdev': [None, 0.351188, 0, None],
'max': [None, 0.8, 0.11, None],
'min': [None, 0.1, 0.11, None]
}
def test_csv_zip():
clean_working_dir()
key = 'fixtures/dmworker/iris-zip.zip'
df = S3File(key)
assert df.is_compressed
assert key == df.get_compressed_name()
local_name = get_local_file_name(df.key)
df.download(local_name)
assert local_name.exists()
assert md5(local_name) == df.etag()
df.compress(local_name)
assert key == df.key
assert df.is_compressed
def test_ts_gz():
clean_working_dir()
key = 'fixtures/dmworker/manual-gz.ts.gz'
df = S3File(key)
assert df.is_compressed
assert key == df.get_compressed_name()
local_name = get_local_file_name(df.key)
df.download(local_name)
assert local_name.exists()
assert md5(local_name) == df.etag()
df.compress(local_name)
assert key == df.key
assert df.is_compressed
df.swap_to_zip()
assert key == df.key
assert df.is_compressed
def test_csv():
clean_working_dir()
key = 'fixtures/dmworker/iris.csv'
df = S3File(key)
assert not df.is_compressed
local_name = get_local_file_name(df.key)
df.download(local_name)
assert local_name.exists()
assert md5(local_name) == df.etag()
md5sum = df.etag()
df.compress(local_name)
zkey = df.key
assert df.is_compressed
clean_working_dir()
local_name = get_local_file_name(df.key)
df.download(local_name)
assert local_name.exists()
with Archive(local_name) as archive:
data = ''.join(x
for x in archive.open_member(archive.get_members()[0]))
assert hashlib.md5(data).hexdigest() == md5sum
def test_csv():
clean_working_dir()
key = 'fixtures/dmworker/iris.csv'
meta = parser.run(key, notify)
assert meta == {
'data_type':
'GENERAL',
'version':
3,
'key':
'fixtures/dmworker/iris.csv.zip',
'size':
18750,
'num_columns':
5,
'data_rows':
150,
'invalid_rows':
0,
'empty_rows':
0,
'delimeter':
'\\s+',
'with_header':
False,
'dtypes': ['f', 'f', 'f', 'f', 'i'],
'classes': [[], [], [], [], []],
'uniques_per_col': [35, 23, 43, 22, 3],
'locked': [True, True, True, True, False],
'histogram': [[9, 23, 14, 27, 22, 20, 18, 6, 5, 6],
[4, 7, 22, 24, 38, 31, 9, 11, 2, 2],
[37, 13, 0, 3, 8, 26, 29, 18, 11, 5],
[41, 8, 1, 7, 8, 33, 6, 23, 9, 14], [50, 50, 50]],
'bins':
[[4.3, 4.66, 5.02, 5.38, 5.74, 6.1, 6.46, 6.82, 7.18, 7.54, 7.9],
[2, 2.24, 2.48, 2.72, 2.96, 3.2, 3.44, 3.68, 3.92, 4.16, 4.4],
[1, 1.59, 2.18, 2.77, 3.36, 3.95, 4.54, 5.13, 5.72, 6.31, 6.9],
[0.1, 0.34, 0.58, 0.82, 1.06, 1.3, 1.54, 1.78, 2.02, 2.26, 2.5],
[0, 0.666667, 1.33333, 2]],
'last_column_info': {
'distrib': {
'1': 0.333333,
'0': 0.333333,
'2': 0.333333
},
'classes': {
'0': 50,
'2': 50,
'1': 50
},
'min': 0,
'max': 2,
'unique': 3
},
'names': ['1', '2', '3', '4', '5'],
'mean': [5.84333, 3.054, 3.75867, 1.19867, 1],
'stdev': [0.828066, 0.433594, 1.76442, 0.763161, 0.819232],
'max': [7.9, 4.4, 6.9, 2.5, 2],
'min': [4.3, 2, 1, 0.1, 0]
}
stat = os.stat(settings.DMWORKER_WORKING_DIR.child(key.replace('/', '_')))
get_key(meta['key']).delete()
meta = parser.run(key, notify)
assert meta == {
'data_type':
'GENERAL',
'version':
3,
'key':
'fixtures/dmworker/iris.csv.zip',
'size':
18750,
'num_columns':
5,
'data_rows':
150,
'invalid_rows':
0,
'empty_rows':
0,
'delimeter':
'\\s+',
'with_header':
False,
'dtypes': ['f', 'f', 'f', 'f', 'i'],
'classes': [[], [], [], [], []],
'uniques_per_col': [35, 23, 43, 22, 3],
'locked': [True, True, True, True, False],
'histogram': [[9, 23, 14, 27, 22, 20, 18, 6, 5, 6],
[4, 7, 22, 24, 38, 31, 9, 11, 2, 2],
[37, 13, 0, 3, 8, 26, 29, 18, 11, 5],
[41, 8, 1, 7, 8, 33, 6, 23, 9, 14], [50, 50, 50]],
'bins':
[[4.3, 4.66, 5.02, 5.38, 5.74, 6.1, 6.46, 6.82, 7.18, 7.54, 7.9],
[2, 2.24, 2.48, 2.72, 2.96, 3.2, 3.44, 3.68, 3.92, 4.16, 4.4],
[1, 1.59, 2.18, 2.77, 3.36, 3.95, 4.54, 5.13, 5.72, 6.31, 6.9],
[0.1, 0.34, 0.58, 0.82, 1.06, 1.3, 1.54, 1.78, 2.02, 2.26, 2.5],
[0, 0.666667, 1.33333, 2]],
'last_column_info': {
'distrib': {
'1': 0.333333,
'0': 0.333333,
'2': 0.333333
},
'classes': {
'1': 50,
'0': 50,
'2': 50
},
'unique': 3,
'max': 2,
'min': 0
},
'names': ['1', '2', '3', '4', '5'],
'mean': [5.84333, 3.054, 3.75867, 1.19867, 1],
'stdev': [0.828066, 0.433594, 1.76442, 0.763161, 0.819232],
'max': [7.9, 4.4, 6.9, 2.5, 2],
'min': [4.3, 2, 1, 0.1, 0]
}
stat2 = os.stat(settings.DMWORKER_WORKING_DIR.child(key.replace('/', '_')))
assert stat.st_ctime == stat2.st_ctime
get_key(meta['key']).delete()
meta = parser.run(key, notify)
assert meta == {
'data_type':
'GENERAL',
'version':
3,
'key':
'fixtures/dmworker/iris.csv.zip',
'size':
18750,
'num_columns':
5,
'data_rows':
150,
'invalid_rows':
0,
'empty_rows':
0,
'delimeter':
'\\s+',
'with_header':
False,
'dtypes': ['f', 'f', 'f', 'f', 'i'],
'classes': [[], [], [], [], []],
'uniques_per_col': [35, 23, 43, 22, 3],
'locked': [True, True, True, True, False],
'histogram': [[9, 23, 14, 27, 22, 20, 18, 6, 5, 6],
[4, 7, 22, 24, 38, 31, 9, 11, 2, 2],
[37, 13, 0, 3, 8, 26, 29, 18, 11, 5],
[41, 8, 1, 7, 8, 33, 6, 23, 9, 14], [50, 50, 50]],
'bins':
[[4.3, 4.66, 5.02, 5.38, 5.74, 6.1, 6.46, 6.82, 7.18, 7.54, 7.9],
[2, 2.24, 2.48, 2.72, 2.96, 3.2, 3.44, 3.68, 3.92, 4.16, 4.4],
[1, 1.59, 2.18, 2.77, 3.36, 3.95, 4.54, 5.13, 5.72, 6.31, 6.9],
[0.1, 0.34, 0.58, 0.82, 1.06, 1.3, 1.54, 1.78, 2.02, 2.26, 2.5],
[0, 0.666667, 1.33333, 2]],
'last_column_info': {
'distrib': {
'0': 0.333333,
'2': 0.333333,
'1': 0.333333
},
'classes': {
'0': 50,
'2': 50,
'1': 50
},
'unique': 3,
'min': 0,
'max': 2
},
'names': ['1', '2', '3', '4', '5'],
'mean': [5.84333, 3.054, 3.75867, 1.19867, 1],
'stdev': [0.828066, 0.433594, 1.76442, 0.763161, 0.819232],
'max': [7.9, 4.4, 6.9, 2.5, 2],
'min': [4.3, 2, 1, 0.1, 0]
}
def test_clean_working_dir():
def create_dir(pwd):
pwd.child('empty_dir').mkdir()
pwd.child('not_empty_dir').mkdir()
pwd.child('not_empty_dir', 'deep_dir').mkdir()
open(pwd.child('not_empty_dir', 'file1'), 'w').close()
open(pwd.child('not_empty_dir', 'file2'), 'w').close()
open(pwd.child('not_empty_dir', 'deep_dir', 'file5'), 'w').close()
open(pwd.child('file3'), 'w').close()
open(pwd.child('file4'), 'w').close()
with tempdir() as td:
create_dir(td)
clean_working_dir(exclude=['not_empty_dir/deep_dir/file5'],
working_dir=td)
assert Path(td).child('not_empty_dir', 'deep_dir', 'file5').exists()
assert len(Path(td).listdir()) == 1
with tempdir() as td:
create_dir(td)
clean_working_dir(
exclude=[Path(td).child('not_empty_dir', 'deep_dir', 'file5')],
working_dir=td)
assert Path(td).child('not_empty_dir', 'deep_dir', 'file5').exists()
assert len(Path(td).listdir()) == 1
with tempdir() as td:
create_dir(td)
clean_working_dir(working_dir=td)
assert Path(td).listdir() == []
with tempdir() as td:
create_dir(td)
clean_working_dir(
exclude=['not_empty_dir/deep_dir/file5', 'not_empty_dir/file1'],
working_dir=td)
assert Path(td).child('not_empty_dir', 'deep_dir', 'file5').exists()
assert Path(td).child('not_empty_dir', 'file1').exists()
assert len(Path(td).listdir()) == 1
with tempdir() as td:
create_dir(td)
clean_working_dir(exclude=['file3'], working_dir=td)
assert Path(td).child('file3').exists()
assert len(Path(td).listdir()) == 1
with tempdir() as td:
create_dir(td)
clean_working_dir(exclude=['file3'], working_dir=td)
assert Path(td).child('file3').exists()
assert len(Path(td).listdir()) == 1
with pytest.raises(ValueError):
with tempdir() as td:
create_dir(td)
clean_working_dir(exclude='file3', working_dir=td)