def mock_worker_checkin(self,
cpus=0,
gpus=0,
memory_bytes=0,
free_disk_bytes=0,
tag=None,
user_id=None):
"""Perform a mock check-in of a new worker."""
worker_id = generate_uuid()
self.bundle_manager._worker_model.worker_checkin(
user_id=user_id
or self.bundle_manager._model.root_user_id, # codalab-owned worker
worker_id=worker_id,
tag=tag,
group_name=None,
cpus=cpus,
gpus=gpus,
memory_bytes=memory_bytes,
free_disk_bytes=free_disk_bytes,
dependencies=[],
shared_file_system=False,
tag_exclusive=False,
exit_after_num_runs=999999999,
is_terminating=False,
)
# Mock a reply from the worker
self.bundle_manager._worker_model.send_json_message = Mock(
return_value=True)
return worker_id
def construct(cls, targets, command, metadata, owner_id, uuid, data_hash,
state):
if not uuid:
uuid = spec_util.generate_uuid()
# Check that targets does not include both keyed and anonymous targets.
if len(targets) > 1 and any(key == '' for key, value in targets):
raise UsageError(
'Must specify keys when packaging multiple targets!')
# List the dependencies of this bundle on its targets.
dependencies = []
for (child_path, (parent_uuid, parent_path)) in targets:
dependencies.append({
'child_uuid': uuid,
'child_path': child_path,
'parent_uuid': parent_uuid,
'parent_path': parent_path,
})
return super(DerivedBundle, cls).construct({
'uuid': uuid,
'bundle_type': cls.BUNDLE_TYPE,
'command': command,
'data_hash': data_hash,
'state': state,
'metadata': metadata,
'dependencies': dependencies,
'owner_id': owner_id,
})
def test_blob_storage_dependency(self):
"""A MakeBundle with a dependency stored on Blob Storage should be made."""
parent = DatasetBundle.construct(
metadata=BASE_METADATA_DATASET_BUNDLE,
owner_id=self.user_id,
uuid=generate_uuid(),
)
bundle = self.create_make_bundle(state=State.STAGED)
bundle.dependencies = [
Dependency({
"parent_uuid": parent.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(parent)
self.save_bundle(bundle)
self.upload_manager.upload_to_bundle_store(
parent,
("contents", BytesIO(FILE_CONTENTS_1.encode())),
git=False,
unpack=True,
use_azure_blob_beta=True,
)
self.make_bundles_and_wait()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.READY)
self.assertEqual(self.read_bundle(bundle, "src"), FILE_CONTENTS_1)
def construct(cls, targets, command, metadata, owner_id, uuid, data_hash, state):
if not uuid: uuid = spec_util.generate_uuid()
# Check that targets does not include both keyed and anonymous targets.
if len(targets) > 1 and any(key == '' for key, value in targets):
raise UsageError('Must specify keys when packaging multiple targets!')
# List the dependencies of this bundle on its targets.
dependencies = []
for (child_path, (parent_uuid, parent_path)) in targets:
dependencies.append({
'child_uuid': uuid,
'child_path': child_path,
'parent_uuid': parent_uuid,
'parent_path': parent_path,
})
return super(DerivedBundle, cls).construct({
'uuid': uuid,
'bundle_type': cls.BUNDLE_TYPE,
'command': command,
'data_hash': data_hash,
'state': state,
'metadata': metadata,
'dependencies': dependencies,
'owner_id': owner_id,
})
def construct(cls, targets, metadata):
uuid = spec_util.generate_uuid()
# Check that targets does not include both keyed and anonymous targets.
if len(targets) > 1 and '' in targets:
raise UsageError('Must specify keys when packaging multiple targets!')
# Support anonymous make bundles with names based on their uuid.
if not metadata['name']:
metadata['name'] = 'make-%s' % (uuid[:cls.NAME_LENGTH],)
# List the dependencies of this bundle on its targets.
dependencies = []
for (child_path, (parent, parent_path)) in targets.iteritems():
dependencies.append({
'child_uuid': uuid,
'child_path': child_path,
'parent_uuid': parent.uuid,
'parent_path': parent_path,
})
return super(MakeBundle, cls).construct({
'uuid': uuid,
'bundle_type': cls.BUNDLE_TYPE,
'command': None,
'data_hash': None,
'state': State.CREATED,
'metadata': metadata,
'dependencies': dependencies,
})
def create_bundle_two_deps(self):
"""Create a bundle with two dependencies. The first dependency is mounted at path "src1"
and the second is mounted at path "src2" of the new bundle.
Returns:
(bundle, parent1, parent2)
"""
parent1 = self.create_run_bundle(state=State.READY)
self.write_bundle(parent1, FILE_CONTENTS_1)
parent2 = self.create_run_bundle(state=State.READY)
self.write_bundle(parent2, FILE_CONTENTS_2)
bundle = MakeBundle.construct(
targets=[],
command='',
metadata=BASE_METADATA_MAKE_BUNDLE,
owner_id=self.user_id,
uuid=generate_uuid(),
state=State.STAGED,
)
bundle.dependencies = [
Dependency({
"parent_uuid": parent1.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src1",
}),
Dependency({
"parent_uuid": parent2.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src2",
}),
]
return bundle, parent1, parent2
def construct(cls, targets, command, metadata, owner_id, uuid=None, data_hash=None, state=State.CREATED):
if not uuid:
uuid = spec_util.generate_uuid()
# Check that targets does not include both keyed and anonymous targets.
if len(targets) > 1 and any(key == "" for key, value in targets):
raise UsageError("Must specify keys when packaging multiple targets!")
# List the dependencies of this bundle on its targets.
dependencies = []
for (child_path, (parent_uuid, parent_path)) in targets:
dependencies.append(
{"child_uuid": uuid, "child_path": child_path, "parent_uuid": parent_uuid, "parent_path": parent_path}
)
return super(MakeBundle, cls).construct(
{
"uuid": uuid,
"bundle_type": cls.BUNDLE_TYPE,
"command": command,
"data_hash": data_hash,
"state": state,
"metadata": metadata,
"dependencies": dependencies,
"owner_id": owner_id,
}
)
def test_linked_dependency(self):
"""A MakeBundle with a linked dependency should be made."""
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(FILE_CONTENTS_1.encode())
tempfile_name = f.name
parent = DatasetBundle.construct(
metadata=dict(BASE_METADATA_DATASET_BUNDLE,
link_url=tempfile_name),
owner_id=self.user_id,
uuid=generate_uuid(),
)
bundle = self.create_make_bundle(state=State.STAGED)
bundle.dependencies = [
Dependency({
"parent_uuid": parent.uuid,
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(parent)
self.save_bundle(bundle)
self.make_bundles_and_wait()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.READY)
self.assertEqual(self.read_bundle(bundle, "src"), FILE_CONTENTS_1)
os.remove(tempfile_name)
def __init__(
self,
docker_image,
initial_command="",
manager=None,
dependencies=[],
bundle_locations={},
verbose=False,
stdout=sys.stdout,
stderr=sys.stderr,
):
# Instantiate a CodaLabManager if one is not passed in
self._manager = manager if manager else CodaLabManager()
self._docker_image = docker_image
self._initial_command = initial_command
InteractiveSession._validate_bundle_locations(bundle_locations,
dependencies)
self._dependencies = dependencies
self._bundle_locations = bundle_locations
self._docker_client = docker.from_env(
timeout=InteractiveSession._MAX_SESSION_TIMEOUT)
self._session_uuid = generate_uuid()
self._verbose = verbose
self._stdout = stdout
self._stderr = stderr
def construct(cls, program_target, input_target, command, metadata):
(program, program_path) = program_target
(input, input_path) = input_target
if not isinstance(program, ProgramBundle):
raise UsageError('%s is not a program!' % (program,))
if not isinstance(input, NamedBundle):
raise UsageError('%s is not a named input!' % (input,))
if not isinstance(command, basestring):
raise UsageError('%r is not a valid command!' % (command,))
uuid = spec_util.generate_uuid()
# Support anonymous run bundles with names based on their uuid.
if not metadata['name']:
metadata['name'] = 'run-%s' % (uuid[:cls.NAME_LENGTH],)
# List the dependencies of this bundle on its targets.
dependencies = []
targets = {'program': program_target, 'input': input_target}
for (child_path, (parent, parent_path)) in targets.iteritems():
dependencies.append({
'child_uuid': uuid,
'child_path': child_path,
'parent_uuid': parent.uuid,
'parent_path': parent_path,
})
return super(RunBundle, cls).construct({
'uuid': uuid,
'bundle_type': cls.BUNDLE_TYPE,
'command': command,
'data_hash': None,
'state': State.CREATED,
'metadata': metadata,
'dependencies': dependencies,
})
def setUp(self):
self.codalab_manager = CodaLabManager()
self.codalab_manager.config['server']['class'] = 'SQLiteModel'
self.bundle_manager = BundleManager(self.codalab_manager)
self.download_manager = self.codalab_manager.download_manager()
self.upload_manager = self.codalab_manager.upload_manager()
# Create a standard user
self.user_id = generate_uuid()
self.bundle_manager._model.add_user(
"codalab_standard",
"*****@*****.**",
"Test",
"User",
"password",
"Stanford",
user_id=self.user_id,
)
# Create a root user
self.root_user_id = self.codalab_manager.root_user_id()
self.bundle_manager._model.add_user(
"codalab_root",
"*****@*****.**",
"Test",
"User",
"password",
"Stanford",
user_id=self.root_user_id,
)
def create_make_bundle(self, state=State.MAKING):
"""Creates a MakeBundle with the given state."""
bundle = MakeBundle.construct(
targets=[],
command='',
metadata=BASE_METADATA_MAKE_BUNDLE,
owner_id=self.user_id,
uuid=generate_uuid(),
state=state,
)
return bundle
def update_in_memory(self, row, strict=False):
items = row.pop('items', None)
if not row.get('uuid'):
row['uuid'] = spec_util.generate_uuid()
super(Worksheet, self).update_in_memory(row)
if items is not None:
self.items = [(item['bundle_uuid'], item['subworksheet_uuid'], item['value'], item['type']) for item in items]
self.last_item_id = max(item['id'] for item in items) if items else -1
else:
self.items = None
self.last_item_id = None
def update_in_memory(self, row, strict=False):
items = row.pop("items", None)
if not row.get("uuid"):
row["uuid"] = spec_util.generate_uuid()
super(Worksheet, self).update_in_memory(row)
if items is not None:
self.items = [
(item["bundle_uuid"], item["subworksheet_uuid"], item["value"], item["type"]) for item in items
]
self.last_item_id = max(item["id"] for item in items) if items else -1
else:
self.items = None
self.last_item_id = None
def update_in_memory(self, row, strict=False):
items = row.pop('items', None)
if not row.get('uuid'):
row['uuid'] = spec_util.generate_uuid()
super(Worksheet, self).update_in_memory(row)
if items is not None:
self.items = [(item['bundle_uuid'], item['subworksheet_uuid'],
item['value'], item['type']) for item in items]
self.last_item_id = max(item['id']
for item in items) if items else -1
else:
self.items = None
self.last_item_id = None
def test_no_permission_parents(self):
"""A bundle with parents that the user doesn't have permission to should
not be staged."""
bundle, parent = self.create_bundle_single_dep()
self.bundle_manager._model.update_bundle(parent,
{"owner_id": generate_uuid()})
self.bundle_manager._stage_bundles()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.FAILED)
self.assertIn("does not have sufficient permissions",
bundle.metadata.failure_message)
def update_in_memory(self, row, strict=False):
items = row.pop('items', None)
if strict:
precondition(items is not None, 'No items: %s' % (row,))
item_sort_keys = [item_sort_key(item) for item in items]
message = 'Worksheet items were not distinct and sorted: %s' % (items,)
precondition(item_sort_keys == sorted(set(item_sort_keys)), message)
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Worksheet, self).update_in_memory(row)
if items is not None:
self.items = [(item['bundle_uuid'], item['value']) for item in items]
self.last_item_id = max(item['id'] for item in items) if items else -1
def update_in_memory(self, row, strict=False):
metadata = row.pop('metadata', None)
dependencies = row.pop('dependencies', None)
if strict:
precondition(metadata is not None, 'No metadata: %s' % (row,))
precondition(dependencies is not None, 'No dependencies: %s' % (row,))
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Bundle, self).update_in_memory(row)
if metadata is not None:
self.metadata = Metadata(self.METADATA_SPECS, metadata)
if dependencies is not None:
self.dependencies = [Dependency(dep) for dep in dependencies]
def update_in_memory(self, row, strict=False):
metadata = row.pop('metadata', None)
dependencies = row.pop('dependencies', None)
if strict:
precondition(metadata is not None, 'No metadata: %s' % (row,))
precondition(dependencies is not None, 'No dependencies: %s' % (row,))
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Bundle, self).update_in_memory(row)
if metadata is not None:
self.metadata = Metadata(self.METADATA_SPECS, metadata)
if dependencies is not None:
self.dependencies = [Dependency(dep) for dep in dependencies]
def test_get_bundle_uuid(self):
tester = self
worksheet_uuid = '0x12345'
missing_name = 'missing_name'
unique_name = 'unique_name'
ambiguous_name = 'ambiguous_name'
bundle_counts = {
missing_name: 0,
unique_name: 1,
ambiguous_name: 2,
}
test_uuids = ['test_uuid_0', 'test_uuid_1', 'test_uuid_2']
test_uuid = test_uuids[0]
class MockBundleModel(object):
def get_bundle_uuids(self, conditions, max_results, count=False):
tester.assertEqual(set(conditions),
set(['name', 'user_id', 'worksheet_uuid']))
name = conditions['name'].replace('%', '')
tester.assertIn(name, bundle_counts)
count = bundle_counts[name]
return [test_uuids[i] for i in range(count)]
model = MockBundleModel()
user_id = None
# Test that get_bundle_uuid is idempotent on generated uuids.
uuid = spec_util.generate_uuid()
self.assertEqual(
uuid,
canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, uuid))
# Test that get_bundle_uuid returns the uuid of a uniquely named bundle.
self.assertEqual(
test_uuid,
canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid,
unique_name))
# Test that get_bundle_uuid raises UsageErrors on missing names.
self.assertRaises(
UsageError,
lambda: canonicalize.get_bundle_uuid(model, user_id,
worksheet_uuid, missing_name),
)
# Test that get_bundle_uuid raises UsageError on specs that can be neither a
# name or a uuid.
self.assertRaises(
UsageError,
lambda: canonicalize.get_bundle_uuid(
model, user_id, worksheet_uuid, 'names have no exclamations!'),
)
def setUp(self):
self.codalab_manager = CodaLabManager()
self.codalab_manager.config['server']['class'] = 'SQLiteModel'
self.bundle_manager = BundleManager(self.codalab_manager)
self.user_id = generate_uuid()
self.bundle_manager._model.add_user(
"codalab",
"*****@*****.**",
"Test",
"User",
"password",
"Stanford",
user_id=self.user_id,
)
def update_in_memory(self, row, strict=False):
items = row.pop('items', None)
self.tags = row.pop('tags', None)
if not row.get('uuid'):
row['uuid'] = spec_util.generate_uuid()
super(Worksheet, self).update_in_memory(row)
if items is not None:
self.items = [
{str(k): v for k, v in item.iteritems()} # Ensure key is string
for item in items
]
self.last_item_id = max(item['id'] for item in items) if items else -1
else:
self.items = None
self.last_item_id = None
def create_run_bundle(self, state=State.CREATED, metadata=None):
"""Creates a RunBundle.
Args:
state: state for the new bundle
metadata: additional metadata to add to the bundle.
"""
bundle = RunBundle.construct(
targets=[],
command='',
metadata=dict(BASE_METADATA, **(metadata or {})),
owner_id=self.user_id,
uuid=generate_uuid(),
state=state,
)
return bundle
def _create_default_groups(self):
'''
Create system-defined groups. This is called by create_tables.
'''
groups = self.batch_get_groups(name='public', user_defined=False)
if len(groups) == 0:
group_dict = self.create_group({'uuid': spec_util.generate_uuid(),
'name': 'public',
'owner_id': None,
'user_defined': False})
else:
group_dict = groups[0]
self.public_group_uuid = group_dict['uuid']
# TODO: find a more systematic way of doing this.
self.root_user_id = '0'
def test_get_spec_uuid(self):
tester = self
missing_name = 'missing_name'
unique_name = 'unique_name'
ambiguous_name = 'ambiguous_name'
bundle_counts = {
missing_name: 0,
unique_name: 1,
ambiguous_name: 2,
}
test_uuids = ['test_uuid_0', 'test_uuid_1', 'test_uuid_2']
test_uuid = test_uuids[0]
class MockBundleModel(object):
def search_bundles(self, **kwargs):
tester.assertEqual(set(kwargs), set(['name']))
tester.assertIn(kwargs['name'], bundle_counts)
count = bundle_counts[kwargs['name']]
return [
type('MockBundle', (object,), {'uuid': test_uuids[i]})
for i in range(count)
]
model = MockBundleModel()
# Test that get_spec_uuid is idempotent on generated uuids.
uuid = spec_util.generate_uuid()
self.assertEqual(uuid, canonicalize.get_spec_uuid(model, uuid))
# Test that get_spec_uuid returns the uuid of a uniquely named bundle.
self.assertEqual(test_uuid, canonicalize.get_spec_uuid(model, unique_name))
# Test that get_spec_uuid raises UsageErrors on missing or ambigious names.
self.assertRaises(
UsageError,
lambda: canonicalize.get_spec_uuid(model, missing_name),
)
self.assertRaises(
UsageError,
lambda: canonicalize.get_spec_uuid(model, ambiguous_name),
)
# Test that get_spec_uuid raises UsageError on specs that can be neither a
# name or a uuid.
self.assertRaises(
UsageError,
lambda: canonicalize.get_spec_uuid(model, 'names have no exclamations!'),
)
def test_get_bundle_uuid(self):
tester = self
worksheet_uuid = '0x12345'
missing_name = 'missing_name'
unique_name = 'unique_name'
ambiguous_name = 'ambiguous_name'
bundle_counts = {
missing_name: 0,
unique_name: 1,
ambiguous_name: 2,
}
test_uuids = ['test_uuid_0', 'test_uuid_1', 'test_uuid_2']
test_uuid = test_uuids[0]
class MockBundleModel(object):
def get_bundle_uuids(self, conditions, max_results, count=False):
tester.assertEqual(set(conditions), set(['name', 'user_id', 'worksheet_uuid']))
name = conditions['name'].replace('%', '')
tester.assertIn(name, bundle_counts)
count = bundle_counts[name]
return [
test_uuids[i]
for i in range(count)
]
model = MockBundleModel()
user_id = None
# Test that get_bundle_uuid is idempotent on generated uuids.
uuid = spec_util.generate_uuid()
self.assertEqual(uuid, canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, uuid))
# Test that get_bundle_uuid returns the uuid of a uniquely named bundle.
self.assertEqual(test_uuid, canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, unique_name))
# Test that get_bundle_uuid raises UsageErrors on missing names.
self.assertRaises(
UsageError,
lambda: canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, missing_name),
)
# Test that get_bundle_uuid raises UsageError on specs that can be neither a
# name or a uuid.
self.assertRaises(
UsageError,
lambda: canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, 'names have no exclamations!'),
)
def test_fail_invalid_dependency_path(self):
"""A MakeBundle with an invalid dependency specified should fail."""
bundle = self.create_make_bundle(state=State.STAGED)
bundle.dependencies = [
Dependency({
"parent_uuid": generate_uuid(),
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(bundle)
self.make_bundles_and_wait()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.FAILED)
self.assertIn("Invalid dependency", bundle.metadata.failure_message)
def test_missing_parent(self):
"""A bundle with a dependency that refers to a nonexistent parent should not
be staged."""
bundle = self.create_run_bundle()
bundle.dependencies = [
Dependency({
"parent_uuid": generate_uuid(),
"parent_path": "",
"child_uuid": bundle.uuid,
"child_path": "src",
})
]
self.save_bundle(bundle)
self.bundle_manager._stage_bundles()
bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
self.assertEqual(bundle.state, State.FAILED)
self.assertIn("Missing parent bundles",
bundle.metadata.failure_message)
def _create_default_groups(self):
'''
Create system-defined groups. This is called by create_tables.
'''
groups = self.batch_get_groups(name='public', user_defined=False)
if len(groups) == 0:
groups = self.batch_get_groups(name='Public', user_defined=False)
if len(groups) == 0:
group_dict = self.create_group({'uuid': spec_util.generate_uuid(),
'name': 'public',
'owner_id': None,
'user_defined': False})
else:
# if there was a group named Public, then rename it.
group_dict = groups[0]
with self.engine.begin() as connection:
connection.execute(cl_group.update().where(
cl_group.c.uuid == group_dict['uuid']
).values({'name': 'public'}))
else:
group_dict = groups[0]
self.public_group_uuid = group_dict['uuid']
def test_delete_nonexistent_group(self):
self.set_current_user('root', '')
uuid = spec_util.generate_uuid()
with self.assertRaises(UsageError):
self.client.rm_group(uuid)
def _create_bundles():
"""
Bulk create bundles.
Query parameters:
- `worksheet`: UUID of the parent worksheet of the new bundle, add to
this worksheet if not detached or shadowing another bundle. The new
bundle also inherits permissions from this worksheet.
- `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
as an item immediately after this bundle in its parent worksheet).
- `detached`: 1 if should not add new bundle to any worksheet,
or 0 otherwise. Default is 0.
- `wait_for_upload`: 1 if the bundle state should be initialized to
"uploading" regardless of the bundle type, or 0 otherwise. Used when
copying bundles from another CodaLab instance, this prevents these new
bundles from being executed by the BundleManager. Default is 0.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
after_sort_key = request.query.get('after_sort_key')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
http.client.BAD_REQUEST,
"Parent worksheet id must be specified as"
"'worksheet' query parameter",
)
# Deserialize bundle fields
bundles = (BundleSchema(strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load(
request.json).data)
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
metadata = bundle.get("metadata", {})
if metadata.get("link_url"):
bundle['state'] = State.READY
elif issubclass(bundle_class, UploadedBundle) or query_get_bool(
'wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
bundle[
'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_items(
worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)],
after_sort_key)
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being created.
bundles = [
bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(bundles).data
def test_not_found(self):
"""Running get_target_info for a nonexistent bundle should raise an error."""
with self.assertRaises(NotFoundError):
target = BundleTarget(generate_uuid(), "")
self.download_manager.get_target_info(target, 0)
def _create_bundles():
"""
Bulk create bundles.
|worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet
if not detached or shadowing another bundle. Also used
to inherit permissions.
|shadow| - the uuid of the bundle to shadow
|detached| - True ('1') if should not add new bundle to any worksheet,
or False ('0') otherwise. Default is False.
|wait_for_upload| - True ('1') if the bundle state should be initialized to
UPLOADING regardless of the bundle type, or False ('0')
otherwise. This prevents run bundles that are being
copied from another instance from being run by the
BundleManager. Default is False.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
httplib.BAD_REQUEST, "Parent worksheet id must be specified as"
"'worksheet' query parameter")
# Deserialize bundle fields
bundles = BundleSchema(
strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS,
).load(request.json).data
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
bundle['state'] = (
State.UPLOADING if issubclass(bundle_class, UploadedBundle)
or query_get_bool('wait_for_upload', False) else State.CREATED)
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
bundles = [bundles_dict[uuid] for uuid in created_uuids]
return BundleSchema(many=True).dump(bundles).data
def update_in_memory(self, row, strict=False):
if strict:
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Group, self).update_in_memory(row)
def test_delete_nonexistent_group(self):
self.set_current_user('root', '')
uuid = spec_util.generate_uuid()
with self.assertRaises(UsageError):
self.client.rm_group(uuid)
def update_in_memory(self, row, strict=False):
if strict:
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Group, self).update_in_memory(row)
