def mock_worker_checkin(self, cpus=0, gpus=0, memory_bytes=0, free_disk_bytes=0, tag=None, user_id=None): """Perform a mock check-in of a new worker.""" worker_id = generate_uuid() self.bundle_manager._worker_model.worker_checkin( user_id=user_id or self.bundle_manager._model.root_user_id, # codalab-owned worker worker_id=worker_id, tag=tag, group_name=None, cpus=cpus, gpus=gpus, memory_bytes=memory_bytes, free_disk_bytes=free_disk_bytes, dependencies=[], shared_file_system=False, tag_exclusive=False, exit_after_num_runs=999999999, is_terminating=False, ) # Mock a reply from the worker self.bundle_manager._worker_model.send_json_message = Mock( return_value=True) return worker_id
def construct(cls, targets, command, metadata, owner_id, uuid, data_hash, state): if not uuid: uuid = spec_util.generate_uuid() # Check that targets does not include both keyed and anonymous targets. if len(targets) > 1 and any(key == '' for key, value in targets): raise UsageError( 'Must specify keys when packaging multiple targets!') # List the dependencies of this bundle on its targets. dependencies = [] for (child_path, (parent_uuid, parent_path)) in targets: dependencies.append({ 'child_uuid': uuid, 'child_path': child_path, 'parent_uuid': parent_uuid, 'parent_path': parent_path, }) return super(DerivedBundle, cls).construct({ 'uuid': uuid, 'bundle_type': cls.BUNDLE_TYPE, 'command': command, 'data_hash': data_hash, 'state': state, 'metadata': metadata, 'dependencies': dependencies, 'owner_id': owner_id, })
def test_blob_storage_dependency(self): """A MakeBundle with a dependency stored on Blob Storage should be made.""" parent = DatasetBundle.construct( metadata=BASE_METADATA_DATASET_BUNDLE, owner_id=self.user_id, uuid=generate_uuid(), ) bundle = self.create_make_bundle(state=State.STAGED) bundle.dependencies = [ Dependency({ "parent_uuid": parent.uuid, "parent_path": "", "child_uuid": bundle.uuid, "child_path": "src", }) ] self.save_bundle(parent) self.save_bundle(bundle) self.upload_manager.upload_to_bundle_store( parent, ("contents", BytesIO(FILE_CONTENTS_1.encode())), git=False, unpack=True, use_azure_blob_beta=True, ) self.make_bundles_and_wait() bundle = self.bundle_manager._model.get_bundle(bundle.uuid) self.assertEqual(bundle.state, State.READY) self.assertEqual(self.read_bundle(bundle, "src"), FILE_CONTENTS_1)
def construct(cls, targets, command, metadata, owner_id, uuid, data_hash, state): if not uuid: uuid = spec_util.generate_uuid() # Check that targets does not include both keyed and anonymous targets. if len(targets) > 1 and any(key == '' for key, value in targets): raise UsageError('Must specify keys when packaging multiple targets!') # List the dependencies of this bundle on its targets. dependencies = [] for (child_path, (parent_uuid, parent_path)) in targets: dependencies.append({ 'child_uuid': uuid, 'child_path': child_path, 'parent_uuid': parent_uuid, 'parent_path': parent_path, }) return super(DerivedBundle, cls).construct({ 'uuid': uuid, 'bundle_type': cls.BUNDLE_TYPE, 'command': command, 'data_hash': data_hash, 'state': state, 'metadata': metadata, 'dependencies': dependencies, 'owner_id': owner_id, })
def construct(cls, targets, metadata): uuid = spec_util.generate_uuid() # Check that targets does not include both keyed and anonymous targets. if len(targets) > 1 and '' in targets: raise UsageError('Must specify keys when packaging multiple targets!') # Support anonymous make bundles with names based on their uuid. if not metadata['name']: metadata['name'] = 'make-%s' % (uuid[:cls.NAME_LENGTH],) # List the dependencies of this bundle on its targets. dependencies = [] for (child_path, (parent, parent_path)) in targets.iteritems(): dependencies.append({ 'child_uuid': uuid, 'child_path': child_path, 'parent_uuid': parent.uuid, 'parent_path': parent_path, }) return super(MakeBundle, cls).construct({ 'uuid': uuid, 'bundle_type': cls.BUNDLE_TYPE, 'command': None, 'data_hash': None, 'state': State.CREATED, 'metadata': metadata, 'dependencies': dependencies, })
def create_bundle_two_deps(self): """Create a bundle with two dependencies. The first dependency is mounted at path "src1" and the second is mounted at path "src2" of the new bundle. Returns: (bundle, parent1, parent2) """ parent1 = self.create_run_bundle(state=State.READY) self.write_bundle(parent1, FILE_CONTENTS_1) parent2 = self.create_run_bundle(state=State.READY) self.write_bundle(parent2, FILE_CONTENTS_2) bundle = MakeBundle.construct( targets=[], command='', metadata=BASE_METADATA_MAKE_BUNDLE, owner_id=self.user_id, uuid=generate_uuid(), state=State.STAGED, ) bundle.dependencies = [ Dependency({ "parent_uuid": parent1.uuid, "parent_path": "", "child_uuid": bundle.uuid, "child_path": "src1", }), Dependency({ "parent_uuid": parent2.uuid, "parent_path": "", "child_uuid": bundle.uuid, "child_path": "src2", }), ] return bundle, parent1, parent2
def construct(cls, targets, command, metadata, owner_id, uuid=None, data_hash=None, state=State.CREATED): if not uuid: uuid = spec_util.generate_uuid() # Check that targets does not include both keyed and anonymous targets. if len(targets) > 1 and any(key == "" for key, value in targets): raise UsageError("Must specify keys when packaging multiple targets!") # List the dependencies of this bundle on its targets. dependencies = [] for (child_path, (parent_uuid, parent_path)) in targets: dependencies.append( {"child_uuid": uuid, "child_path": child_path, "parent_uuid": parent_uuid, "parent_path": parent_path} ) return super(MakeBundle, cls).construct( { "uuid": uuid, "bundle_type": cls.BUNDLE_TYPE, "command": command, "data_hash": data_hash, "state": state, "metadata": metadata, "dependencies": dependencies, "owner_id": owner_id, } )
def test_linked_dependency(self): """A MakeBundle with a linked dependency should be made.""" with tempfile.NamedTemporaryFile(delete=False) as f: f.write(FILE_CONTENTS_1.encode()) tempfile_name = f.name parent = DatasetBundle.construct( metadata=dict(BASE_METADATA_DATASET_BUNDLE, link_url=tempfile_name), owner_id=self.user_id, uuid=generate_uuid(), ) bundle = self.create_make_bundle(state=State.STAGED) bundle.dependencies = [ Dependency({ "parent_uuid": parent.uuid, "parent_path": "", "child_uuid": bundle.uuid, "child_path": "src", }) ] self.save_bundle(parent) self.save_bundle(bundle) self.make_bundles_and_wait() bundle = self.bundle_manager._model.get_bundle(bundle.uuid) self.assertEqual(bundle.state, State.READY) self.assertEqual(self.read_bundle(bundle, "src"), FILE_CONTENTS_1) os.remove(tempfile_name)
def __init__( self, docker_image, initial_command="", manager=None, dependencies=[], bundle_locations={}, verbose=False, stdout=sys.stdout, stderr=sys.stderr, ): # Instantiate a CodaLabManager if one is not passed in self._manager = manager if manager else CodaLabManager() self._docker_image = docker_image self._initial_command = initial_command InteractiveSession._validate_bundle_locations(bundle_locations, dependencies) self._dependencies = dependencies self._bundle_locations = bundle_locations self._docker_client = docker.from_env( timeout=InteractiveSession._MAX_SESSION_TIMEOUT) self._session_uuid = generate_uuid() self._verbose = verbose self._stdout = stdout self._stderr = stderr
def construct(cls, program_target, input_target, command, metadata): (program, program_path) = program_target (input, input_path) = input_target if not isinstance(program, ProgramBundle): raise UsageError('%s is not a program!' % (program,)) if not isinstance(input, NamedBundle): raise UsageError('%s is not a named input!' % (input,)) if not isinstance(command, basestring): raise UsageError('%r is not a valid command!' % (command,)) uuid = spec_util.generate_uuid() # Support anonymous run bundles with names based on their uuid. if not metadata['name']: metadata['name'] = 'run-%s' % (uuid[:cls.NAME_LENGTH],) # List the dependencies of this bundle on its targets. dependencies = [] targets = {'program': program_target, 'input': input_target} for (child_path, (parent, parent_path)) in targets.iteritems(): dependencies.append({ 'child_uuid': uuid, 'child_path': child_path, 'parent_uuid': parent.uuid, 'parent_path': parent_path, }) return super(RunBundle, cls).construct({ 'uuid': uuid, 'bundle_type': cls.BUNDLE_TYPE, 'command': command, 'data_hash': None, 'state': State.CREATED, 'metadata': metadata, 'dependencies': dependencies, })
def setUp(self): self.codalab_manager = CodaLabManager() self.codalab_manager.config['server']['class'] = 'SQLiteModel' self.bundle_manager = BundleManager(self.codalab_manager) self.download_manager = self.codalab_manager.download_manager() self.upload_manager = self.codalab_manager.upload_manager() # Create a standard user self.user_id = generate_uuid() self.bundle_manager._model.add_user( "codalab_standard", "*****@*****.**", "Test", "User", "password", "Stanford", user_id=self.user_id, ) # Create a root user self.root_user_id = self.codalab_manager.root_user_id() self.bundle_manager._model.add_user( "codalab_root", "*****@*****.**", "Test", "User", "password", "Stanford", user_id=self.root_user_id, )
def create_make_bundle(self, state=State.MAKING): """Creates a MakeBundle with the given state.""" bundle = MakeBundle.construct( targets=[], command='', metadata=BASE_METADATA_MAKE_BUNDLE, owner_id=self.user_id, uuid=generate_uuid(), state=state, ) return bundle
def update_in_memory(self, row, strict=False): items = row.pop('items', None) if not row.get('uuid'): row['uuid'] = spec_util.generate_uuid() super(Worksheet, self).update_in_memory(row) if items is not None: self.items = [(item['bundle_uuid'], item['subworksheet_uuid'], item['value'], item['type']) for item in items] self.last_item_id = max(item['id'] for item in items) if items else -1 else: self.items = None self.last_item_id = None
def update_in_memory(self, row, strict=False): items = row.pop("items", None) if not row.get("uuid"): row["uuid"] = spec_util.generate_uuid() super(Worksheet, self).update_in_memory(row) if items is not None: self.items = [ (item["bundle_uuid"], item["subworksheet_uuid"], item["value"], item["type"]) for item in items ] self.last_item_id = max(item["id"] for item in items) if items else -1 else: self.items = None self.last_item_id = None
def test_no_permission_parents(self): """A bundle with parents that the user doesn't have permission to should not be staged.""" bundle, parent = self.create_bundle_single_dep() self.bundle_manager._model.update_bundle(parent, {"owner_id": generate_uuid()}) self.bundle_manager._stage_bundles() bundle = self.bundle_manager._model.get_bundle(bundle.uuid) self.assertEqual(bundle.state, State.FAILED) self.assertIn("does not have sufficient permissions", bundle.metadata.failure_message)
def update_in_memory(self, row, strict=False): items = row.pop('items', None) if strict: precondition(items is not None, 'No items: %s' % (row,)) item_sort_keys = [item_sort_key(item) for item in items] message = 'Worksheet items were not distinct and sorted: %s' % (items,) precondition(item_sort_keys == sorted(set(item_sort_keys)), message) if 'uuid' not in row: row['uuid'] = spec_util.generate_uuid() super(Worksheet, self).update_in_memory(row) if items is not None: self.items = [(item['bundle_uuid'], item['value']) for item in items] self.last_item_id = max(item['id'] for item in items) if items else -1
def update_in_memory(self, row, strict=False): metadata = row.pop('metadata', None) dependencies = row.pop('dependencies', None) if strict: precondition(metadata is not None, 'No metadata: %s' % (row,)) precondition(dependencies is not None, 'No dependencies: %s' % (row,)) if 'uuid' not in row: row['uuid'] = spec_util.generate_uuid() super(Bundle, self).update_in_memory(row) if metadata is not None: self.metadata = Metadata(self.METADATA_SPECS, metadata) if dependencies is not None: self.dependencies = [Dependency(dep) for dep in dependencies]
def test_get_bundle_uuid(self): tester = self worksheet_uuid = '0x12345' missing_name = 'missing_name' unique_name = 'unique_name' ambiguous_name = 'ambiguous_name' bundle_counts = { missing_name: 0, unique_name: 1, ambiguous_name: 2, } test_uuids = ['test_uuid_0', 'test_uuid_1', 'test_uuid_2'] test_uuid = test_uuids[0] class MockBundleModel(object): def get_bundle_uuids(self, conditions, max_results, count=False): tester.assertEqual(set(conditions), set(['name', 'user_id', 'worksheet_uuid'])) name = conditions['name'].replace('%', '') tester.assertIn(name, bundle_counts) count = bundle_counts[name] return [test_uuids[i] for i in range(count)] model = MockBundleModel() user_id = None # Test that get_bundle_uuid is idempotent on generated uuids. uuid = spec_util.generate_uuid() self.assertEqual( uuid, canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, uuid)) # Test that get_bundle_uuid returns the uuid of a uniquely named bundle. self.assertEqual( test_uuid, canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, unique_name)) # Test that get_bundle_uuid raises UsageErrors on missing names. self.assertRaises( UsageError, lambda: canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, missing_name), ) # Test that get_bundle_uuid raises UsageError on specs that can be neither a # name or a uuid. self.assertRaises( UsageError, lambda: canonicalize.get_bundle_uuid( model, user_id, worksheet_uuid, 'names have no exclamations!'), )
def setUp(self): self.codalab_manager = CodaLabManager() self.codalab_manager.config['server']['class'] = 'SQLiteModel' self.bundle_manager = BundleManager(self.codalab_manager) self.user_id = generate_uuid() self.bundle_manager._model.add_user( "codalab", "*****@*****.**", "Test", "User", "password", "Stanford", user_id=self.user_id, )
def update_in_memory(self, row, strict=False): items = row.pop('items', None) self.tags = row.pop('tags', None) if not row.get('uuid'): row['uuid'] = spec_util.generate_uuid() super(Worksheet, self).update_in_memory(row) if items is not None: self.items = [ {str(k): v for k, v in item.iteritems()} # Ensure key is string for item in items ] self.last_item_id = max(item['id'] for item in items) if items else -1 else: self.items = None self.last_item_id = None
def create_run_bundle(self, state=State.CREATED, metadata=None): """Creates a RunBundle. Args: state: state for the new bundle metadata: additional metadata to add to the bundle. """ bundle = RunBundle.construct( targets=[], command='', metadata=dict(BASE_METADATA, **(metadata or {})), owner_id=self.user_id, uuid=generate_uuid(), state=state, ) return bundle
def _create_default_groups(self): ''' Create system-defined groups. This is called by create_tables. ''' groups = self.batch_get_groups(name='public', user_defined=False) if len(groups) == 0: group_dict = self.create_group({'uuid': spec_util.generate_uuid(), 'name': 'public', 'owner_id': None, 'user_defined': False}) else: group_dict = groups[0] self.public_group_uuid = group_dict['uuid'] # TODO: find a more systematic way of doing this. self.root_user_id = '0'
def test_get_spec_uuid(self): tester = self missing_name = 'missing_name' unique_name = 'unique_name' ambiguous_name = 'ambiguous_name' bundle_counts = { missing_name: 0, unique_name: 1, ambiguous_name: 2, } test_uuids = ['test_uuid_0', 'test_uuid_1', 'test_uuid_2'] test_uuid = test_uuids[0] class MockBundleModel(object): def search_bundles(self, **kwargs): tester.assertEqual(set(kwargs), set(['name'])) tester.assertIn(kwargs['name'], bundle_counts) count = bundle_counts[kwargs['name']] return [ type('MockBundle', (object,), {'uuid': test_uuids[i]}) for i in range(count) ] model = MockBundleModel() # Test that get_spec_uuid is idempotent on generated uuids. uuid = spec_util.generate_uuid() self.assertEqual(uuid, canonicalize.get_spec_uuid(model, uuid)) # Test that get_spec_uuid returns the uuid of a uniquely named bundle. self.assertEqual(test_uuid, canonicalize.get_spec_uuid(model, unique_name)) # Test that get_spec_uuid raises UsageErrors on missing or ambigious names. self.assertRaises( UsageError, lambda: canonicalize.get_spec_uuid(model, missing_name), ) self.assertRaises( UsageError, lambda: canonicalize.get_spec_uuid(model, ambiguous_name), ) # Test that get_spec_uuid raises UsageError on specs that can be neither a # name or a uuid. self.assertRaises( UsageError, lambda: canonicalize.get_spec_uuid(model, 'names have no exclamations!'), )
def test_get_bundle_uuid(self): tester = self worksheet_uuid = '0x12345' missing_name = 'missing_name' unique_name = 'unique_name' ambiguous_name = 'ambiguous_name' bundle_counts = { missing_name: 0, unique_name: 1, ambiguous_name: 2, } test_uuids = ['test_uuid_0', 'test_uuid_1', 'test_uuid_2'] test_uuid = test_uuids[0] class MockBundleModel(object): def get_bundle_uuids(self, conditions, max_results, count=False): tester.assertEqual(set(conditions), set(['name', 'user_id', 'worksheet_uuid'])) name = conditions['name'].replace('%', '') tester.assertIn(name, bundle_counts) count = bundle_counts[name] return [ test_uuids[i] for i in range(count) ] model = MockBundleModel() user_id = None # Test that get_bundle_uuid is idempotent on generated uuids. uuid = spec_util.generate_uuid() self.assertEqual(uuid, canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, uuid)) # Test that get_bundle_uuid returns the uuid of a uniquely named bundle. self.assertEqual(test_uuid, canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, unique_name)) # Test that get_bundle_uuid raises UsageErrors on missing names. self.assertRaises( UsageError, lambda: canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, missing_name), ) # Test that get_bundle_uuid raises UsageError on specs that can be neither a # name or a uuid. self.assertRaises( UsageError, lambda: canonicalize.get_bundle_uuid(model, user_id, worksheet_uuid, 'names have no exclamations!'), )
def test_fail_invalid_dependency_path(self): """A MakeBundle with an invalid dependency specified should fail.""" bundle = self.create_make_bundle(state=State.STAGED) bundle.dependencies = [ Dependency({ "parent_uuid": generate_uuid(), "parent_path": "", "child_uuid": bundle.uuid, "child_path": "src", }) ] self.save_bundle(bundle) self.make_bundles_and_wait() bundle = self.bundle_manager._model.get_bundle(bundle.uuid) self.assertEqual(bundle.state, State.FAILED) self.assertIn("Invalid dependency", bundle.metadata.failure_message)
def test_missing_parent(self): """A bundle with a dependency that refers to a nonexistent parent should not be staged.""" bundle = self.create_run_bundle() bundle.dependencies = [ Dependency({ "parent_uuid": generate_uuid(), "parent_path": "", "child_uuid": bundle.uuid, "child_path": "src", }) ] self.save_bundle(bundle) self.bundle_manager._stage_bundles() bundle = self.bundle_manager._model.get_bundle(bundle.uuid) self.assertEqual(bundle.state, State.FAILED) self.assertIn("Missing parent bundles", bundle.metadata.failure_message)
def _create_default_groups(self): ''' Create system-defined groups. This is called by create_tables. ''' groups = self.batch_get_groups(name='public', user_defined=False) if len(groups) == 0: groups = self.batch_get_groups(name='Public', user_defined=False) if len(groups) == 0: group_dict = self.create_group({'uuid': spec_util.generate_uuid(), 'name': 'public', 'owner_id': None, 'user_defined': False}) else: # if there was a group named Public, then rename it. group_dict = groups[0] with self.engine.begin() as connection: connection.execute(cl_group.update().where( cl_group.c.uuid == group_dict['uuid'] ).values({'name': 'public'})) else: group_dict = groups[0] self.public_group_uuid = group_dict['uuid']
def test_delete_nonexistent_group(self): self.set_current_user('root', '') uuid = spec_util.generate_uuid() with self.assertRaises(UsageError): self.client.rm_group(uuid)
def _create_bundles(): """ Bulk create bundles. Query parameters: - `worksheet`: UUID of the parent worksheet of the new bundle, add to this worksheet if not detached or shadowing another bundle. The new bundle also inherits permissions from this worksheet. - `shadow`: UUID of the bundle to "shadow" (the new bundle will be added as an item immediately after this bundle in its parent worksheet). - `detached`: 1 if should not add new bundle to any worksheet, or 0 otherwise. Default is 0. - `wait_for_upload`: 1 if the bundle state should be initialized to "uploading" regardless of the bundle type, or 0 otherwise. Used when copying bundles from another CodaLab instance, this prevents these new bundles from being executed by the BundleManager. Default is 0. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') after_sort_key = request.query.get('after_sort_key') detached = query_get_bool('detached', default=False) if worksheet_uuid is None: abort( http.client.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter", ) # Deserialize bundle fields bundles = (BundleSchema(strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load( request.json).data) # Check for all necessary permissions worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id metadata = bundle.get("metadata", {}) if metadata.get("link_url"): bundle['state'] = State.READY elif issubclass(bundle_class, UploadedBundle) or query_get_bool( 'wait_for_upload', False): bundle['state'] = State.UPLOADING else: bundle['state'] = State.CREATED bundle[ 'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) # Inherit worksheet permissions group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid) set_bundle_permissions([{ 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions]) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: local.model.add_worksheet_items( worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)], after_sort_key) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted # right after being created. bundles = [ bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict ] return BundleSchema(many=True).dump(bundles).data
def test_not_found(self): """Running get_target_info for a nonexistent bundle should raise an error.""" with self.assertRaises(NotFoundError): target = BundleTarget(generate_uuid(), "") self.download_manager.get_target_info(target, 0)
def _create_bundles(): """ Bulk create bundles. |worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet if not detached or shadowing another bundle. Also used to inherit permissions. |shadow| - the uuid of the bundle to shadow |detached| - True ('1') if should not add new bundle to any worksheet, or False ('0') otherwise. Default is False. |wait_for_upload| - True ('1') if the bundle state should be initialized to UPLOADING regardless of the bundle type, or False ('0') otherwise. This prevents run bundles that are being copied from another instance from being run by the BundleManager. Default is False. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') detached = query_get_bool('detached', default=False) if worksheet_uuid is None: abort( httplib.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter") # Deserialize bundle fields bundles = BundleSchema( strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS, ).load(request.json).data # Check for all necessary permissions worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id bundle['state'] = ( State.UPLOADING if issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False) else State.CREATED) bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) # Inherit worksheet permissions group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid) set_bundle_permissions([{ 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions]) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: local.model.add_worksheet_item( worksheet_uuid, worksheet_util.bundle_item(bundle_uuid)) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order bundles = [bundles_dict[uuid] for uuid in created_uuids] return BundleSchema(many=True).dump(bundles).data
def update_in_memory(self, row, strict=False): if strict: if 'uuid' not in row: row['uuid'] = spec_util.generate_uuid() super(Group, self).update_in_memory(row)