def test_create_context():
context_name = '__test__'
assert context_name not in api.ls_contexts(), 'Context exists'
api.context(context_name)
assert context_name in api.ls_contexts(), 'Test context does exists'
api.delete_context(context_name=context_name)
assert context_name not in api.ls_contexts(), 'Test context exists'
def test_independent_context():
context_1_name = '__test_context_1__'
context_2_name = '__test_context_2__'
api.context(context_1_name)
api.context(context_2_name)
api.apply(context_1_name, ContextTest)
assert len(api.search(context_1_name)) == 1, 'Only one bundle should be in context one'
assert len(api.search(context_2_name)) == 0, 'Context two should be empty'
api.delete_context(context_name=context_1_name)
api.delete_context(context_name=context_2_name)
assert context_1_name not in api.ls_contexts(), 'Contexts should be removed'
assert context_2_name not in api.ls_contexts(), 'Contexts should be removed'
def setup():
if TEST_CONTEXT in api.ls_contexts():
api.delete_context(context_name=TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
def test_add_with_treat_as_bundle():
api.delete_context(TEST_CONTEXT)
api.context(context_name=TEST_CONTEXT)
# Setup moto s3 resources
s3_client = boto3.client('s3')
s3_resource = boto3.resource('s3')
s3_resource.create_bucket(Bucket=TEST_BUCKET)
# Make sure bucket is empty
objects = s3_client.list_objects(Bucket=TEST_BUCKET)
assert 'Contents' not in objects, 'Bucket should be empty'
# Bind remote context
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL, force=True)
# Run test pipeline
api.apply(TEST_CONTEXT, CIP)
# Push bundles to remote
for bundle_name in ['a', 'b', 'c']:
assert api.get(TEST_CONTEXT,
bundle_name) is not None, 'Bundle should exist'
api.commit(TEST_CONTEXT, bundle_name)
api.push(TEST_CONTEXT, bundle_name)
# Blow away context and recreate
api.delete_context(TEST_CONTEXT)
assert TEST_CONTEXT not in api.ls_contexts()
api.context(context_name=TEST_CONTEXT)
api.remote(TEST_CONTEXT, TEST_REMOTE, TEST_BUCKET_URL, force=True)
assert api.search(TEST_CONTEXT) == [], 'Context should be empty'
# Pull bundles from remote
api.pull(TEST_CONTEXT)
# Make sure all bundle meta data comes down but data remains in S3
for bundle_name in ['a', 'b', 'c']:
bundle = api.get(TEST_CONTEXT, bundle_name)
assert bundle is not None, 'Bundle should exist'
data_path = bundle.data['file'][0]
assert data_path.startswith('s3://'), 'Data should be in S3'
# Rerun pipeline
api.apply(TEST_CONTEXT, BIP, params={'n': 100}, incremental_pull=True)
# Make sure all bundles exist. Bundles a and b should have local paths
for bundle_name in ['a', 'b', 'c']:
bundle = api.get(TEST_CONTEXT, bundle_name)
assert bundle is not None, 'Bundle should exist'
data_path = bundle.data['file'][0]
if bundle_name in ['a', 'b']:
assert not data_path.startswith('s3://'), 'Data should be local'
else:
assert data_path.startswith('s3://'), 'Data should be in S3'
api.delete_context(TEST_CONTEXT)