def test_run_export_no_data():
with tempdatasetdir() as dsa:
dsa.write_config({'cmd': 'echo hi && (echo muahaha >&2)'})
with pytest.raises(Exception):
dsa._run_export(dataset.new_parcel_id())
assert sum(1 for x in dsa.data_path.glob('*')) == 0
assert sum(1 for x in dsa.incomplete_path.glob('*')) == 0
assert next(dsa.log_path.glob('*.out')).read_text() == 'hi\n'
assert next(dsa.log_path.glob('*.err')).read_text() == 'muahaha\n'
def test_ingest_path():
with tempdatasetdir() as dsa:
path = dsa.path.joinpath('foo.txt')
path.write_text('hello')
parcel_id = dataset.new_parcel_id()
dsa.ingest_path(path, parcel_id)
assert sum(1 for x in dsa.incomplete_path.glob('*')) == 0
assert (
dsa.data_path.joinpath(f'{parcel_id}.txt').read_text() == 'hello')
assert dsa.read_metrics()[parcel_id]['success'] == 'Y'
def test_run_export():
with tempdatasetdir() as dsa:
dsa.write_config({'cmd': 'echo hi > $PARCEL_PATH.txt'})
parcel_id = dataset.new_parcel_id()
dsa._run_export(parcel_id)
assert sum(1 for x in dsa.incomplete_path.glob('*')) == 0
assert (
dsa.data_path.joinpath(f'{parcel_id}.txt').read_text() == 'hi\n')
assert dsa.log_path.joinpath(f'{parcel_id}.out').read_text() == ''
assert dsa.log_path.joinpath(f'{parcel_id}.err').read_text() == ''
def test_ingest_path_git():
with tempdatasetdir(git=True) as dsa:
path = dsa.path.joinpath('foo.txt')
path.write_text('hello')
parcel_id = dataset.new_parcel_id()
dsa.ingest_path(path, parcel_id)
assert sum(1 for x in dsa.incomplete_path.glob('*')) == 0
assert (
dsa.data_path.joinpath(f'{parcel_id}.txt').read_text() == 'hello')
assert dsa.read_metrics()[parcel_id]['success'] == 'Y'
repo = Repo(dsa.path)
assert ([b.name for b in repo.head.commit.tree['data']
] == [f'{parcel_id}.txt'])
def test_ingest_path_bad_path():
with tempdatasetdir() as dsa:
path = dsa.path.joinpath('foo.txt')
with pytest.raises(FileNotFoundError):
dsa.ingest_path(path, dataset.new_parcel_id())
def test_run_export_no_cmd():
with tempdatasetdir() as dsa:
dsa._run_export(dataset.new_parcel_id())
assert sum(1 for x in dsa.data_path.glob('*')) == 0
assert sum(1 for x in dsa.incomplete_path.glob('*')) == 0