def create_bundle_two_deps(self):
"""Create a bundle with two dependencies. The first dependency is mounted at path "src1"
and the second is mounted at path "src2" of the new bundle.
Returns:
(bundle, parent1, parent2)
"""
parent1 = self.create_run_bundle(state=State.READY)
self.write_bundle(parent1, FILE_CONTENTS_1)
parent2 = self.create_run_bundle(state=State.READY)
self.write_bundle(parent2, FILE_CONTENTS_2)
bundle = MakeBundle.construct(
targets=[],
command='',
metadata=BASE_METADATA_MAKE_BUNDLE,
owner_id=self.user_id,
uuid=generate_uuid(),
state=State.STAGED,
)
bundle.dependencies = [
Dependency({
"parent_uuid": parent1.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src1",
}),
Dependency({
"parent_uuid": parent2.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src2",
}),
]
return bundle, parent1, parent2
def test_blob_storage_dependency(self):
"""A MakeBundle with a dependency stored on Blob Storage should be made."""
parent = DatasetBundle.construct(
metadata=BASE_METADATA_DATASET_BUNDLE,
owner_id=self.user_id,
uuid=generate_uuid(),
)
bundle = self.create_make_bundle(state=State.STAGED)
bundle.dependencies = [
Dependency({
"parent_uuid": parent.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(parent)
self.save_bundle(bundle)
self.upload_manager.upload_to_bundle_store(
parent,
("contents", BytesIO(FILE_CONTENTS_1.encode())),
git=False,
unpack=True,
use_azure_blob_beta=True,
)
self.make_bundles_and_wait()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.READY)
self.assertEqual(self.read_bundle(bundle, "src"), FILE_CONTENTS_1)
def test_linked_dependency(self):
"""A MakeBundle with a linked dependency should be made."""
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(FILE_CONTENTS_1.encode())
tempfile_name = f.name
parent = DatasetBundle.construct(
metadata=dict(BASE_METADATA_DATASET_BUNDLE,
link_url=tempfile_name),
owner_id=self.user_id,
uuid=generate_uuid(),
)
bundle = self.create_make_bundle(state=State.STAGED)
bundle.dependencies = [
Dependency({
"parent_uuid": parent.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(parent)
self.save_bundle(bundle)
self.make_bundles_and_wait()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.READY)
self.assertEqual(self.read_bundle(bundle, "src"), FILE_CONTENTS_1)
os.remove(tempfile_name)
def create_bundle_single_dep(self,
parent_state=State.READY,
bundle_state=State.CREATED,
bundle_type=RunBundle):
"""Creates a bundle with a single dependency, which is mounted at path "src" of the
new bundle.
Args:
parent_state: State of the parent bundle. Defaults to State.READY.
bundle_state: State of the new bundle. Defaults to State.CREATED.
bundle_type: Type of child bundle to create; valid values are RunBundle and MakeBundle. Defaults to RunBundle.
Returns:
(bundle, parent)
"""
parent = self.create_run_bundle(parent_state)
self.write_bundle(parent, FILE_CONTENTS_1)
bundle = (self.create_run_bundle(bundle_state) if bundle_type
== RunBundle else self.create_make_bundle(bundle_state))
bundle.dependencies = [
Dependency({
"parent_uuid": parent.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
return bundle, parent
def update_in_memory(self, row, strict=False):
metadata = row.pop('metadata', None)
dependencies = row.pop('dependencies', None)
if strict:
precondition(metadata is not None, 'No metadata: %s' % (row,))
precondition(dependencies is not None, 'No dependencies: %s' % (row,))
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Bundle, self).update_in_memory(row)
if metadata is not None:
self.metadata = Metadata(self.METADATA_SPECS, metadata)
if dependencies is not None:
self.dependencies = [Dependency(dep) for dep in dependencies]
def test_fail_invalid_dependency_path(self):
"""A MakeBundle with an invalid dependency specified should fail."""
bundle = self.create_make_bundle(state=State.STAGED)
bundle.dependencies = [
Dependency({
"parent_uuid": generate_uuid(),
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(bundle)
self.make_bundles_and_wait()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.FAILED)
self.assertIn("Invalid dependency", bundle.metadata.failure_message)
def test_missing_parent(self):
"""A bundle with a dependency that refers to a nonexistent parent should not
be staged."""
bundle = self.create_run_bundle()
bundle.dependencies = [
Dependency({
"parent_uuid": generate_uuid(),
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(bundle)
self.bundle_manager._stage_bundles()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.FAILED)
self.assertIn("Missing parent bundles",
bundle.metadata.failure_message)