def test_remote_no_push_non_managed_s3():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
s3_resource.create_bucket(Bucket=TEST_BUCKET_OTHER)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
objects = s3_client.list_objects(Bucket=TEST_BUCKET_OTHER)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
# Apply
api.apply(TEST_CONTEXT, NonManagedS3)
print(api.cat(TEST_CONTEXT, 'b2'))
# Local context should not contain file if a remote exists.
b = api.search(TEST_CONTEXT, human_name='b2')[0]
assert not os.path.exists(
b.data['file']
[0]), 'Non Managed S3 file w/ remote should be copied to remote'
assert b.data['file'][0].startswith("s3://")
def test_add_with_treat_as_bundle():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL, force=True)
# Try to run the pipeline - should fail
try:
# Run test pipeline
api.apply(TEST_CONTEXT, CPush, incremental_push=True)
except Exception as e:
pass
# Get objects from remote
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
keys = [o['Key'] for o in objects['Contents']]
keys = [key.split('/')[-1] for key in keys]
# Make sure files exist in S3
for output_file in ['a.txt', 'b.txt']:
assert output_file in keys, 'Pipeline should have pushed file'
api.delete_context(TEST_CONTEXT)
def test():
""" Purpose of this test is to have one task that produces a bundle.
And another task that requires it.
1.) Create external dep -- also creates PreMaker_auf_datamaker
dsdt apply - - test_external_bundle.DataMaker --int_array '[1000,2000,3000]'
2.) Remove Premaker_auf_datamaker
dsdt rm PreMaker_auf_datamaker
3.) Try to run Root -- it should find DataMaker but not re-create it or PreMaker_auf_datamaker
"""
api.context(TEST_CONTEXT)
api.apply(TEST_CONTEXT,
'-',
'-',
'DataMaker',
params={'int_array': '[1000,2000,3000]'})
b = api.get(TEST_CONTEXT, 'PreMaker_auf_datamaker')
assert (b is not None)
b.rm()
api.apply(TEST_CONTEXT, '-', '-', 'Root')
b = api.get(TEST_CONTEXT, 'PreMaker_auf_root')
assert (b is not None)
api.delete_context(TEST_CONTEXT)
def deprecated_data_as_bundle_not_csv(tmpdir):
# Create Context
api.context(TEST_CONTEXT)
# Create test .txt file
test_txt_path = os.path.join(str(tmpdir), 'test.txt')
with open(test_txt_path, 'w') as f:
f.write('this should not create a bundle')
# Assert the txt file exists
assert os.path.exists(test_txt_path)
# Try to add file to the bundle
with pytest.raises(AssertionError) as ex:
api.add(TEST_CONTEXT,
'bad_path',
test_txt_path,
treat_file_as_bundle=True)
# Assert Exited with error code of 1
assert ex.type == AssertionError
# Make sure bundle does not exist
assert api.get(
TEST_CONTEXT,
'test_file_as_bundle_txt_file') is None, 'Bundle should not exist'
api.delete_context(TEST_CONTEXT)
def test_bundle_link_localization_with_remote():
""" Test the ability to localize and delocalize individual links
Note: you can only localize / de-localize a closed bundle.
create bundles.
test w/o a remote
"""
_ = _setup()
b = create_local_file_bundle("sharkies")
b.push()
# all local files. then delocalize each in turn.
for i, f in enumerate(b.data):
assert f.startswith("/")
b.delocalize(f)
assert b.data[i].startswith("s3://")
# bring it all back
for i, f in enumerate(b.data):
assert b.data[i].startswith(
"s3://"
) # testing if we can both s3 and local paths in a link frame
b.localize(f)
assert b.data[i].startswith("/")
api.delete_context(TEST_CONTEXT)
def test_bundle_push_delocalize():
""" Test Bundle.push(delocalize)
Test if we can push individually, and see that the files actualize to s3 paths.
"""
s3_client = _setup()
bundles = {}
for i in range(3):
name = "shark{}".format(i)
bundles[name] = create_local_file_bundle(name)
bundles[name].push(delocalize=True)
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' in objects, 'Bucket should not be empty'
assert len(objects['Contents']) > 0, 'Bucket should not be empty'
# Check for delocalization
for b in bundles.values():
for i, f in enumerate(b.data):
assert f.startswith("s3://")
# Might as well test the push and pull worked.
api.rm(TEST_CONTEXT, rm_all=True)
api.pull(TEST_CONTEXT, localize=True)
found_bundles = {b.name: b for b in api.search(TEST_CONTEXT)}
for n, b in found_bundles.items():
for i, f in enumerate(b.data):
assert md5_file(f) == b.tags["f{}".format(i)]
api.delete_context(TEST_CONTEXT)
def X_test_api_push_delocalize():
""" Test api.push(delocalize)
Unfortunately, moto fails on testing this with MP, so removing test.
"""
_ = _setup()
bundles = {}
for i in range(3):
name = "shark{}".format(i)
bundles[name] = create_local_file_bundle(name)
# Moto keeps s3 xfers in memory, so multiprocessing will succeed
# But when the subprocess exits, the files will disappear
# So we cannot see the result of the push (b/c of moto in memory s3),
# but the delocalize should remove the data.
api.push(
TEST_CONTEXT,
delocalize=True) # push and remote all data, then pull and localize.
for b in bundles.values():
for i, f in enumerate(b.data):
assert f.startswith("s3://")
api.delete_context(TEST_CONTEXT)
def test_bundle_link_localization_no_remote():
""" Test the ability to localize and delocalize individual links
Note: you can only localize / de-localize a closed bundle.
create bundles.
test w/o a remote
"""
_ = _setup(remote=False)
b = create_local_file_bundle("sharkies")
# all local files. then delocalize each in turn.
errors = 0
for i, f in enumerate(b.data):
assert f.startswith("/")
try:
b.delocalize(f)
except AssertionError as ae:
print("Caught an expected assertion error -- {}".format(ae))
errors += 1
assert b.data[i].startswith("/") # should have failed
assert errors == len(b.data)
api.delete_context(TEST_CONTEXT)
def test_fast_push():
s3_client = _setup()
bundles = {}
for i in range(3):
name = "shark{}".format(i)
bundles[name] = create_local_file_bundle(name)
bundles[name].push(
) # need this to put the bundles in the remote b/c of moto mp issues
# Moto keeps s3 xfers in memory, so multiprocessing will succeed
# But when the subprocess exits, the files will disappear
# This is basically useless in a test.
api.push(TEST_CONTEXT) # push and remote all data, then pull and localize.
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' in objects, 'Bucket should not be empty'
assert len(objects['Contents']) > 0, 'Bucket should not be empty'
api.rm(TEST_CONTEXT, rm_all=True)
api.pull(TEST_CONTEXT, localize=True)
found_bundles = {b.name: b for b in api.search(TEST_CONTEXT)}
for n, b in found_bundles.items():
for i, f in enumerate(b.data):
assert md5_file(f) == b.tags["f{}".format(i)]
api.delete_context(TEST_CONTEXT)
def test_remote_push_managed_s3():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
# Apply
api.apply(TEST_CONTEXT, ManagedS3, incremental_push=True)
assert not os.path.exists(api.search(TEST_CONTEXT, human_name='b4')[0].data['file'][0]), \
'Managed S3 file should not be copied to local'
# Get objects from remote
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
keys = [o['Key'] for o in objects['Contents']]
keys = [key.split('/')[-1] for key in keys]
# Make sure files exist in S3
for output_file in ['test.parquet']:
assert output_file in keys, 'Pipeline should have pushed file'
def test_add_remote():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3', region_name='us-east-1')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context with just bucket
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
with api.Bundle(TEST_CONTEXT) as b:
b.name = 'output'
b.add_data([1, 3, 5])
b.commit()
b.push()
# Bind remote to new context with bucket and key
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_KEY_URL)
with api.Bundle(TEST_CONTEXT) as b:
b.name = 'output'
b.add_data([1, 3, 5])
b.commit()
b.push()
api.delete_context(TEST_CONTEXT)
def test():
""" This tests if mark_force works for tasks.
We have two tasks. One depends on the other. The upstream is marked
mark_force and should always run.
"""
def run_and_get(name, do_ext=False):
api.apply(TEST_CONTEXT, 'A_2', params={'set_ext_dep': do_ext})
b = api.get(TEST_CONTEXT, 'B')
print("Run {}: b.creation_date {} b.uuid {}".format(
name, b.creation_date, b.uuid))
return b
api.delete_context(TEST_CONTEXT)
api.context(TEST_CONTEXT)
b = run_and_get("One")
first_uuid = b.uuid
b = run_and_get("Two")
assert (first_uuid != b.uuid)
second_uuid = b.uuid
b = run_and_get("Three", do_ext=True)
assert (second_uuid == b.uuid)
api.delete_context(TEST_CONTEXT)
def test_pull(run_test):
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
bucket = s3_resource.Bucket(TEST_BUCKET)
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
assert len(api.search(TEST_CONTEXT)) == 0, 'Context should be empty'
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
api.apply(TEST_CONTEXT, RemoteTest)
bundle = api.get(TEST_CONTEXT, 'remote_test')
assert bundle.data == 'Hello'
bundle.commit()
bundle.push()
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' in objects, 'Bucket should not be empty'
assert len(objects['Contents']) > 0, 'Bucket should not be empty'
api.delete_context(context_name=TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
api.pull(TEST_CONTEXT)
pulled_bundles = api.search(TEST_CONTEXT)
assert len(pulled_bundles) > 0, 'Pulled bundles down'
assert pulled_bundles[0].data == 'Hello', 'Bundle contains correct data'
bucket.objects.all().delete()
bucket.delete()
def test_add_remote_fail():
error = None
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3', region_name='us-east-1')
# Bind remote context with just bucket
try:
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
except Exception as e:
error = e
finally:
assert (type(error) == RuntimeError)
# Bind remote to new context with bucket and key
try:
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_KEY_URL)
except Exception as e:
error = e
finally:
assert (type(error) == RuntimeError)
api.delete_context(TEST_CONTEXT)
def test():
"""
Returns:
"""
api.context(TEST_CONTEXT)
api.remote(TEST_CONTEXT, TEST_CONTEXT, REMOTE_URL, force=True)
with api.Bundle(TEST_CONTEXT, TEST_NAME, owner=getpass.getuser()) as b:
for i in range(3):
with b.add_file('output_{}'.format(i)).open('w') as of:
of.write("some text for the {} file".format(i))
b.commit().push()
b.rm()
b.pull(localize=False)
api.apply(TEST_CONTEXT,
'-',
'test_output',
'ConsumeExtDep',
incremental_pull=True)
api.delete_context(TEST_CONTEXT, remote=True)
def test(run_test):
""" Purpose of this test is to have one task that produces a bundle.
And another task that requires it.
1.) Run DataMaker which runs PreMaker
2.) Assert that those ran, and remove PreMaker
3.) run Root_1 which needs DataMaker (external dep) and PreMaker
4.) assert that premaker re-ran and root ran successfully (getting external dependency)
"""
api.context(TEST_CONTEXT)
api.apply(TEST_CONTEXT,
DataMaker,
params={'int_array': [1000, 2000, 3000]})
b = api.get(TEST_CONTEXT, 'PreMaker')
assert (b is not None)
pm_uuid = b.uuid
b.rm()
api.apply(TEST_CONTEXT, Root_1)
b = api.get(TEST_CONTEXT, 'PreMaker')
assert (b is not None)
assert (b.uuid != pm_uuid)
b = api.get(TEST_CONTEXT, 'Root_1')
assert (b is not None)
api.delete_context(TEST_CONTEXT)
def test_push():
api.context(context_name=TEST_CONTEXT)
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
bucket = s3_resource.Bucket(TEST_BUCKET)
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
assert len(api.search(TEST_CONTEXT)) == 0, 'Context should be empty'
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL, force=True)
api.apply(TEST_CONTEXT, 'RemoteTest')
bundle = api.get(TEST_CONTEXT, 'remote_test')
assert bundle.data == 'Hello'
bundle.commit()
bundle.push()
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' in objects, 'Bucket should not be empty'
assert len(objects['Contents']) > 0, 'Bucket should not be empty'
bucket.objects.all().delete()
bucket.delete()
api.delete_context(context_name=TEST_CONTEXT)
def test_create_context():
context_name = '__test__'
assert context_name not in api.ls_contexts(), 'Context exists'
api.context(context_name)
assert context_name in api.ls_contexts(), 'Test context does exists'
api.delete_context(context_name=context_name)
assert context_name not in api.ls_contexts(), 'Test context exists'
def test_no_remote_no_push_managed_s3():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
assert len(api.search(TEST_CONTEXT)) == 0, 'Context should be empty'
with pytest.raises(Exception) as e:
api.apply(TEST_CONTEXT, ManagedS3)
def test_non_managed_local():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
assert len(api.search(TEST_CONTEXT)) == 0, 'Context should be empty'
api.apply(TEST_CONTEXT, NonManagedLocal)
assert len(
api.search(TEST_CONTEXT)) == 1, 'Only one bundle should be present'
print(api.cat(TEST_CONTEXT, 'b1'))
assert os.path.exists(api.search(TEST_CONTEXT, human_name='b1')[0].data['file'][0]), \
'Local file should be present in bundle'
def context():
try:
print('ensuring disdat is initialized')
common.DisdatConfig.init()
except:
print('disdat already initialized, no worries...')
print('creating temporary local context')
context = uuid.uuid1().hex
api.context(context)
yield context
print('deleting temporary local context')
api.delete_context(context)
def test_independent_context():
context_1_name = '__test_context_1__'
context_2_name = '__test_context_2__'
api.context(context_1_name)
api.context(context_2_name)
api.apply(context_1_name, ContextTest)
assert len(api.search(context_1_name)) == 1, 'Only one bundle should be in context one'
assert len(api.search(context_2_name)) == 0, 'Context two should be empty'
api.delete_context(context_name=context_1_name)
api.delete_context(context_name=context_2_name)
assert context_1_name not in api.ls_contexts(), 'Contexts should be removed'
assert context_2_name not in api.ls_contexts(), 'Contexts should be removed'
def test_remote_no_push_managed_s3():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
with pytest.raises(Exception) as e:
api.apply(TEST_CONTEXT, ManagedS3)
def _setup(remote=True):
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3', region_name='us-east-1')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context
if remote:
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL)
return s3_client
def test_single_file(tmpdir):
# Create Context
api.context(TEST_CONTEXT)
# Create test .csv file
test_csv_path = os.path.join(str(tmpdir), 'test.csv')
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
df.to_csv(test_csv_path)
# Assert csv_file_exits
assert os.path.exists(test_csv_path)
# Add the file to the bundle
api.add(TEST_CONTEXT, 'test_single_file', test_csv_path)
# Retrieve the bundle
b = api.get(TEST_CONTEXT, 'test_single_file')
# Assert the bundles contain the same data
bundle_hash, file_hash = get_hash(b.data), get_hash(test_csv_path)
assert bundle_hash == file_hash, 'Hashes do not match'
# Test with tags
tag = {'test': 'tag'}
api.add(TEST_CONTEXT, 'test_single_file', test_csv_path, tags=tag)
# Retrieve the bundle
b = api.get(TEST_CONTEXT, 'test_single_file')
# Assert the bundles contain the same data
bundle_hash, file_hash = get_hash(b.data), get_hash(test_csv_path)
assert bundle_hash == file_hash, 'Hashes do not match'
assert b.tags == tag, 'Tags do not match'
# Remove test .csv
os.remove(test_csv_path)
# Assert that data still remains in the bundle
assert api.get(TEST_CONTEXT,
'test_single_file') is not None, 'Bundle should exist'
api.delete_context(TEST_CONTEXT)
def test_add_bad_path(tmpdir):
# Create Context
api.context(TEST_CONTEXT)
# Create path to csv file but don't create file
test_csv_path = os.path.join(str(tmpdir), 'test.csv')
# Assert csv file does not exist
assert not os.path.exists(test_csv_path)
# Try to add file to the bundle
with pytest.raises(AssertionError) as ex:
api.add(TEST_CONTEXT, 'bad_path', test_csv_path)
# Assert Exited with error code of 1
assert ex.type == AssertionError
# Make sure bundle does not exist
assert api.get(TEST_CONTEXT, 'test_file_as_bundle_txt_file') is None, 'Bundle should not exist'
api.delete_context(TEST_CONTEXT)
def test_no_remote_push_non_managed_s3():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
s3_resource.create_bucket(Bucket=TEST_BUCKET_OTHER)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
objects = s3_client.list_objects(Bucket=TEST_BUCKET_OTHER)
assert 'Contents' not in objects, 'Bucket should be empty'
api.apply(TEST_CONTEXT, NonManagedS3, incremental_push=True)
print(api.cat(TEST_CONTEXT, 'b2'))
assert len(api.search(TEST_CONTEXT)) == 1, 'One bundle should be present'
assert os.path.exists(api.search(TEST_CONTEXT, human_name='b2')[0].data['file'][0]), \
'Non Managed S3 file should be copied to local'
def run_test():
# Remove test context before running test
setup()
yield
api.delete_context(context_name=TEST_CONTEXT)
def setup():
if TEST_CONTEXT in api.ls_contexts():
api.delete_context(context_name=TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
def test_add_directory(tmpdir):
# Create Context
api.context(TEST_CONTEXT)
# Directory Structure
# - test.csv
# - second/test_1.txt
# - second/test_2.txt
# - second/third/test_3.txt
# - second/third/test_4.txt
level_1 = ''
level_2 = os.path.join(level_1, 'second')
os.mkdir(os.path.join(str(tmpdir), level_2))
level_3 = os.path.join(level_2, 'third')
os.mkdir(os.path.join(str(tmpdir), level_3))
# Dictionary to hold paths
path_dict = {}
# Create files and save paths
test_csv_name = 'test.csv'
test_csv_path = os.path.join(level_1, test_csv_name)
test_csv_abs_path = os.path.join(str(tmpdir), test_csv_path)
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
df.to_csv(test_csv_abs_path)
path_dict[test_csv_name] = (test_csv_abs_path, test_csv_path.split('/'))
test_text_1_name = 'test_1.txt'
test_text_1_path = os.path.join(level_2, test_text_1_name)
test_text_name_1_abs_path = os.path.join(str(tmpdir), test_text_1_path)
with open(test_text_name_1_abs_path, 'w') as f:
f.write('Hello!')
path_dict[test_text_1_name] = (test_text_name_1_abs_path,
test_text_1_path.split('/'))
test_text_2_name = 'test_2.txt'
test_text_2_path = os.path.join(level_2, test_text_2_name)
test_text_name_2_abs_path = os.path.join(str(tmpdir), test_text_2_path)
with open(test_text_name_2_abs_path, 'w') as f:
f.write('Hello!')
path_dict[test_text_2_name] = (test_text_name_2_abs_path,
test_text_2_path.split('/'))
test_text_3_name = 'test_3.txt'
test_text_3_path = os.path.join(level_3, test_text_3_name)
test_text_name_3_abs_path = os.path.join(str(tmpdir), test_text_3_path)
with open(test_text_name_3_abs_path, 'w') as f:
f.write('Third Hello!')
path_dict[test_text_3_name] = (test_text_name_3_abs_path,
test_text_3_path.split('/'))
test_text_4_name = 'test_4.txt'
test_text_4_path = os.path.join(level_3, test_text_4_name)
test_text_name_4_abs_path = os.path.join(str(tmpdir), test_text_4_path)
with open(test_text_name_4_abs_path, 'w') as f:
f.write('Third World!')
path_dict[test_text_4_name] = (test_text_name_4_abs_path,
test_text_4_path.split('/'))
# Assert files exist
assert os.path.exists(test_csv_abs_path)
assert os.path.exists(test_text_name_1_abs_path)
assert os.path.exists(test_text_name_2_abs_path)
assert os.path.exists(test_text_name_3_abs_path)
assert os.path.exists(test_text_name_4_abs_path)
# Add the directory to the bundle
api.add(TEST_CONTEXT, 'test_directory', str(tmpdir))
# Assert check sums are the same
b = api.get(TEST_CONTEXT, 'test_directory')
for f in b.data:
bundle_file_name = f.split('/')[-1]
local_abs_path, local_split_path = path_dict[bundle_file_name]
# Make sure paths match
assert get_hash(f) == get_hash(local_abs_path), 'Hashes do not match'
bundle_path = os.path.join(*f.split('/')[-len(local_split_path):])
local_path = os.path.join(*local_split_path)
assert local_path == bundle_path, 'Bundle should have the same directory structure'
# Add the directory to the bundle with tags
tag = {'test': 'tag'}
api.add(TEST_CONTEXT, 'test_directory', str(tmpdir), tags=tag)
# Assert check sums are the same
b = api.get(TEST_CONTEXT, 'test_directory')
for f in b.data:
bundle_file_name = f.split('/')[-1]
local_abs_path, local_split_path = path_dict[bundle_file_name]
# Make sure paths match
assert get_hash(f) == get_hash(local_abs_path), 'Hashes do not match'
# Make sure directory structure stays the same
local_path = os.path.join(*local_split_path)
bundle_path = os.path.join(*f.split('/')[-len(local_split_path):])
assert local_path == bundle_path, 'Bundle should have the same directory structure'
# Make sure tags exist
assert b.tags == tag, 'Tags do not match'
api.delete_context(TEST_CONTEXT)
