def replace_items(worksheet_uuid): """ Replace worksheet items with 'ids' with new 'items'. The new items will be inserted after the after_sort_key """ worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) ids = request.json.get('ids', []) item_type = request.json.get('item_type', 'markup') after_sort_key = request.json.get('after_sort_key') # Default to process only markup items. if item_type == "markup": items = [ worksheet_util.markup_item(item) for item in request.json.get('items', []) ] elif item_type == "bundle": items = [ worksheet_util.bundle_item(item) for item in request.json.get('items', []) ] elif item_type == "directive": # Note: for directives, the item should not include the preceding "%" symbol items = [ worksheet_util.directive_item(item) for item in request.json.get('items', []) ] local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks): bundle_type = info['bundle_type'] if 'uuid' in info: existing = True construct_args = self.bundle_info_to_construct_args(info) else: existing = False construct_args = {'metadata': info['metadata']} metadata = construct_args['metadata'] message = 'Invalid upload bundle_type: %s' % (bundle_type,) if not existing: precondition(bundle_type in UPLOADED_TYPES, message) bundle_subclass = get_bundle_subclass(bundle_type) if not existing: self.validate_user_metadata(bundle_subclass, metadata) # Upload the given path and record additional metadata from the upload. if path: (data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks) metadata.update(bundle_store_metadata) precondition(construct_args.get('data_hash', data_hash) == data_hash, \ 'Provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash)) construct_args['data_hash'] = data_hash # Set the owner construct_args['owner_id'] = self._current_user_id() bundle = bundle_subclass.construct(**construct_args) self.model.save_bundle(bundle) if worksheet_uuid: self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid)) # TODO: don't fail if don't have permissions return bundle.uuid
def interpret_search(query): """ Input: specification of a bundle search query. Output: worksheet items based on the result of issuing the search query. """ # Perform search keywords = rest_util.resolve_owner_in_keywords(query['keywords']) bundle_uuids = local.model.search_bundle_uuids(request.user.user_id, keywords) # Single number, just print it out... if not isinstance(bundle_uuids, list): return interpret_items(query['schemas'], [markup_item(str(bundle_uuids))]) # Set display items = [directive_item(('display', ) + tuple(query['display']))] # Show bundles bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: items.append(bundle_item(bundle_infos[bundle_uuid])) # Finally, interpret the items return interpret_items(query['schemas'], items)
def derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid): ''' For both make and run bundles. Add the resulting bundle to the given worksheet_uuid. ''' check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False)) bundle_uuid = self._derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid) # Add to worksheet self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid)) return bundle_uuid
def expand_raw_item(raw_item): """ Raw items that include searches must be expanded into more raw items. Input: Raw item. Output: Array of raw items. If raw item does not need expanding, this returns an 1-length array that contains original raw item, otherwise it contains the search result. You do not need to call resolve_items_into_infos on the returned raw_items. """ (bundle_info, subworksheet_info, value_obj, item_type, id, sort_key) = raw_item is_search = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'search' is_wsearch = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'wsearch' if is_search or is_wsearch: command = get_command(value_obj) keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) if search_result['is_aggregate']: # Add None's for the 'id' and 'sort_key' of the tuple, since these # items are not really worksheet items. raw_items.append( markup_item(str(search_result['result'])) + (None, None)) else: bundle_uuids = search_result['result'] bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: # Since bundle UUID's are queried first, we can't assume a UUID exists in # the subsequent bundle info query. if bundle_uuid in bundle_infos: raw_items.append( bundle_item(bundle_infos[bundle_uuid]) + (None, None)) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append( subworksheet_item(worksheet_info) + (None, None)) return raw_items else: return [raw_item]
def derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid): ''' For both make and run bundles. Add the resulting bundle to the given worksheet_uuid (optional). ''' bundle_subclass = get_bundle_subclass(bundle_type) self.validate_user_metadata(bundle_subclass, metadata) owner_id = self._current_user_id() bundle = bundle_subclass.construct(targets=targets, command=command, metadata=metadata, owner_id=owner_id) self.model.save_bundle(bundle) if worksheet_uuid: self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid)) # TODO: don't fail if don't have permissions return bundle.uuid
def perform_search_query(value_obj): """ Perform a search query and return the resulting raw items. Input: directive that is tokenized by formatting.string_to_tokens(), such as formatting.string_to_tokens("search 0x .limit=100") """ command = get_command(value_obj) is_search = command == 'search' is_wsearch = command == 'wsearch' if is_search or is_wsearch: keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) if search_result['is_aggregate']: # Add None's for the 'id' and 'sort_key' of the tuple, since these # items are not really worksheet items. raw_items.append( markup_item(str(search_result['result'])) + (None, None)) else: bundle_uuids = search_result['result'] bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: # Since bundle UUID's are queried first, we can't assume a UUID exists in # the subsequent bundle info query. if bundle_uuid in bundle_infos: raw_items.append( bundle_item(bundle_infos[bundle_uuid]) + (None, None)) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append( subworksheet_item(worksheet_info) + (None, None)) return raw_items else: # Not a search query return []
def expand_raw_item(raw_item): """ Raw items that include searches must be expanded into more raw items. Input: Raw item. Output: Array of raw items. If raw item does not need expanding, this returns an 1-length array that contains original raw item, otherwise it contains the search result. You do not need to call resolve_items_into_infos on the returned raw_items. """ (bundle_info, subworksheet_info, value_obj, item_type) = raw_item is_search = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'search' is_wsearch = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'wsearch' if is_search or is_wsearch: command = get_command(value_obj) keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundle_uuids( request.user.user_id, keywords) if not isinstance(search_result, list): raw_items.append(markup_item(str(search_result))) else: bundle_uuids = search_result bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: raw_items.append(bundle_item(bundle_infos[bundle_uuid])) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append(subworksheet_item(worksheet_info)) return raw_items else: return [raw_item]
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks, exclude_patterns, add_to_worksheet): check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False)) bundle_type = info['bundle_type'] if 'uuid' in info: existing = True construct_args = self.bundle_info_to_construct_args(info) else: existing = False construct_args = {'metadata': info['metadata']} metadata = construct_args['metadata'] message = 'Invalid upload bundle_type: %s' % (bundle_type,) if not existing: precondition(bundle_type in UPLOADED_TYPES, message) bundle_subclass = get_bundle_subclass(bundle_type) if not existing: self.validate_user_metadata(bundle_subclass, metadata) # Upload the given path and record additional metadata from the upload. if path: (data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns) metadata.update(bundle_store_metadata) if construct_args.get('data_hash', data_hash) != data_hash: print >>sys.stderr, 'ERROR: provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash) construct_args['data_hash'] = data_hash # Set the owner construct_args['owner_id'] = self._current_user_id() bundle = bundle_subclass.construct(**construct_args) self.model.save_bundle(bundle) # Inherit properties of worksheet self._bundle_inherit_workheet_permissions(bundle.uuid, worksheet_uuid) # Add to worksheet if add_to_worksheet: self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid)) return bundle.uuid
def expand_raw_item(raw_item): """ Raw items that include searches must be expanded into more raw items. Input: Raw item. Output: Array of raw items. If raw item does not need expanding, this returns an 1-length array that contains original raw item, otherwise it contains the search result. You do not need to call resolve_items_into_infos on the returned raw_items. """ (bundle_info, subworksheet_info, value_obj, item_type) = raw_item is_search = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'search' is_wsearch = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'wsearch' if is_search or is_wsearch: command = get_command(value_obj) keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) if search_result['is_aggregate']: raw_items.append(markup_item(str(search_result['result']))) else: bundle_uuids = search_result['result'] bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: raw_items.append(bundle_item(bundle_infos[bundle_uuid])) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append(subworksheet_item(worksheet_info)) return raw_items else: return [raw_item]
def add_with_prelude_items(old_bundle_uuid, new_bundle_uuid): ''' Add new_bundle_uuid to the current worksheet (along with the occurrences on the worksheet). Also add the prelude (items that occur right before old_bundle_uuid on a worksheet). Note that new_bundle_uuid might get multiple times. ''' host_worksheet_uuids = self.model.get_host_worksheet_uuids([old_bundle_uuid])[old_bundle_uuid] if len(host_worksheet_uuids) > 0: # Choose a single worksheet. if worksheet_uuid in host_worksheet_uuids: # If current worksheet is one of them, favor that one. host_worksheet_uuid = worksheet_uuid else: # Choose an arbitrary one host_worksheet_uuid = host_worksheet_uuids[0] if host_worksheet_uuid in worksheet_cache: worksheet_info = worksheet_cache[host_worksheet_uuid] else: worksheet_info = worksheet_cache[host_worksheet_uuid] = \ self.get_worksheet_info(host_worksheet_uuid, fetch_items=True) # Look for items that appear right before the old_bundle_uuid collect_items = [] for item in worksheet_info['items']: (bundle_info, subworkheet_info, value_obj, type) = item if type == worksheet_util.TYPE_BUNDLE and bundle_info['uuid'] == old_bundle_uuid: # Ended in the target old_bundle_uuid, flush the prelude gathered so far. for item2 in collect_items: self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2)) self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) collect_items = [] elif type == worksheet_util.TYPE_MARKUP and value_obj == '': collect_items = [] elif type == worksheet_util.TYPE_BUNDLE: collect_items = [] else: collect_items.append(item)
def replace_items(worksheet_uuid): """ Replace worksheet items with 'ids' with new 'items'. The new items will be inserted after the after_sort_key """ worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) ids = request.json.get('ids', []) item_type = request.json.get('item_type', 'markup') after_sort_key = request.json.get('after_sort_key') # Default to process only markup items. if item_type == "markup": items = [ worksheet_util.markup_item(item) for item in request.json.get('items', []) ] elif item_type == "bundle": items = [ worksheet_util.bundle_item(item) for item in request.json.get('items', []) ] local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def _create_bundles(): """ Bulk create bundles. |worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet if not detached or shadowing another bundle. Also used to inherit permissions. |shadow| - the uuid of the bundle to shadow |detached| - True ('1') if should not add new bundle to any worksheet, or False ('0') otherwise. Default is False. |wait_for_upload| - True ('1') if the bundle state should be initialized to UPLOADING regardless of the bundle type, or False ('0') otherwise. This prevents run bundles that are being copied from another instance from being run by the BundleManager. Default is False. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') detached = query_get_bool('detached', default=False) if worksheet_uuid is None: abort( httplib.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter") # Deserialize bundle fields bundles = BundleSchema( strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS, ).load(request.json).data # Check for all necessary permissions worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id bundle['state'] = ( State.UPLOADING if issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False) else State.CREATED) bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) # Inherit worksheet permissions group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid) set_bundle_permissions([{ 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions]) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: local.model.add_worksheet_item( worksheet_uuid, worksheet_util.bundle_item(bundle_uuid)) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order bundles = [bundles_dict[uuid] for uuid in created_uuids] return BundleSchema(many=True).dump(bundles).data
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run): ''' old_inputs: list of bundle uuids old_output: bundle uuid that we produced new_inputs: list of bundle uuids that are analogous to old_inputs new_output_name: name of the bundle to create to be analogous to old_output (possibly None) worksheet_uuid: add newly created bundles to this worksheet depth: how far to do a BFS up from old_output. shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets. ''' check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False)) #print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name) # Build the graph (get all the infos). # If old_output is given, look at ancestors of old_output until we # reached some depth. If it's not given, we first get all the # descendants first, and then get their ancestors. infos = {} # uuid -> bundle info if old_output: bundle_uuids = [old_output] else: bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth) all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order for _ in range(depth): new_bundle_uuids = [] for bundle_uuid in bundle_uuids: if bundle_uuid in infos: continue # Already visited info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid) for dep in info['dependencies']: parent_uuid = dep['parent_uuid'] if parent_uuid not in infos: new_bundle_uuids.append(parent_uuid) all_bundle_uuids = new_bundle_uuids + all_bundle_uuids bundle_uuids = new_bundle_uuids # Make sure we have read access to all the bundles involved here. check_bundles_have_read_permission(self.model, self._current_user(), list(infos.keys())) # Now go recursively create the bundles. old_to_new = {} # old_uuid -> new_uuid downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid) created_uuids = set() # set of uuids which were newly created plan = [] # sequence of (old, new) bundle infos to make for old, new in zip(old_inputs, new_inputs): old_to_new[old] = new downstream.add(old) # Return corresponding new_bundle_uuid def recurse(old_bundle_uuid): if old_bundle_uuid in old_to_new: #print old_bundle_uuid, 'cached' return old_to_new[old_bundle_uuid] # Don't have any more information (because we probably hit the maximum depth) if old_bundle_uuid not in infos: #print old_bundle_uuid, 'no information' return old_bundle_uuid # Get information about the old bundle. info = infos[old_bundle_uuid] new_dependencies = [{ 'parent_uuid': recurse(dep['parent_uuid']), 'parent_path': dep['parent_path'], 'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test 'child_path': dep['child_path'] } for dep in info['dependencies']] # If there are no inputs or if we're downstream of any inputs, we need to make a new bundle. lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output) downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies']) if lone_output or downstream_of_inputs: # Now create a new bundle that mimics the old bundle. # Only change the name if the output name is supplied. old_bundle_name = info['metadata']['name'] new_info = copy.deepcopy(info) new_metadata = new_info['metadata'] if new_output_name: if old_bundle_uuid == old_output: new_metadata['name'] = new_output_name else: # Just make up a name heuristically new_metadata['name'] = new_output_name + '-' + info['metadata']['name'] # Remove all the automatically generated keys cls = get_bundle_subclass(new_info['bundle_type']) for spec in cls.METADATA_SPECS: if spec.generated and spec.key in new_metadata: new_metadata.pop(spec.key) # Set the targets targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies] if dry_run: new_bundle_uuid = None else: if new_info['bundle_type'] not in ('make', 'run'): raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid) new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \ targets, new_info['command'], new_metadata, worksheet_uuid) new_info['uuid'] = new_bundle_uuid plan.append((info, new_info)) downstream.add(old_bundle_uuid) created_uuids.add(new_bundle_uuid) else: new_bundle_uuid = old_bundle_uuid old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it return new_bundle_uuid if old_output: recurse(old_output) else: # Don't have a particular output we're targetting, so just create # new versions of all the uuids. for uuid in all_bundle_uuids: recurse(uuid) # Add to worksheet if not dry_run: if shadow: # Add each new bundle in the "shadow" of the old_bundle (right after it). for old_bundle_uuid, new_bundle_uuid in old_to_new.items(): if new_bundle_uuid in created_uuids: # Only add novel bundles self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid) else: def newline(): self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item('')) # A prelude of a bundle on a worksheet is the set of items that occur right before it (markup, directives, etc.) # Let W be the first worksheet containing the old_inputs[0]. # Add all items on that worksheet that appear in old_to_new along with their preludes. # For items not on this worksheet, add them at the end (instead of making them floating). if old_output: anchor_uuid = old_output elif len(old_inputs) > 0: anchor_uuid = old_inputs[0] host_worksheet_uuids = self.model.get_host_worksheet_uuids([anchor_uuid])[anchor_uuid] new_bundle_uuids_added = set() skipped = True # Whether there were items that we didn't include in the prelude (in which case we want to put '') if len(host_worksheet_uuids) > 0: # Choose a single worksheet. if worksheet_uuid in host_worksheet_uuids: # If current worksheet is one of them, favor that one. host_worksheet_uuid = worksheet_uuid else: # Choose an arbitrary one (in the future, have a better way of canonicalizing). host_worksheet_uuid = host_worksheet_uuids[0] # Fetch the worksheet worksheet_info = self.get_worksheet_info(host_worksheet_uuid, fetch_items=True) prelude_items = [] # The prelude that we're building up for item in worksheet_info['items']: (bundle_info, subworkheet_info, value_obj, item_type) = item just_added = False if item_type == worksheet_util.TYPE_BUNDLE: old_bundle_uuid = bundle_info['uuid'] if old_bundle_uuid in old_to_new: # Flush the prelude gathered so far. new_bundle_uuid = old_to_new[old_bundle_uuid] if new_bundle_uuid in created_uuids: # Only add novel bundles # Stand in for things skipped (this is important so directives have proper extent). if skipped: newline() # Add prelude and items for item2 in prelude_items: self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2)) self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) new_bundle_uuids_added.add(new_bundle_uuid) just_added = True if (item_type == worksheet_util.TYPE_MARKUP and value_obj != '') or item_type == worksheet_util.TYPE_DIRECTIVE: prelude_items.append(item) # Include in prelude skipped = False else: prelude_items = [] # Reset skipped = not just_added # Add the bundles that haven't been added yet for info, new_info in plan: new_bundle_uuid = new_info['uuid'] if new_bundle_uuid not in new_bundle_uuids_added: if skipped: newline() skipped = False self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) return plan
def add_worksheet_item(self, worksheet_uuid, bundle_uuid): """ Add bundle uuid to the given worksheet. """ self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def add_worksheet_item(self, worksheet_uuid, bundle_uuid): self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def add_worksheet_item(self, worksheet_uuid, bundle_uuid): """ Add bundle uuid to the given worksheet. """ self.client.add_worksheet_item( worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def _create_bundles(): """ Bulk create bundles. Query parameters: - `worksheet`: UUID of the parent worksheet of the new bundle, add to this worksheet if not detached or shadowing another bundle. The new bundle also inherits permissions from this worksheet. - `shadow`: UUID of the bundle to "shadow" (the new bundle will be added as an item immediately after this bundle in its parent worksheet). - `detached`: 1 if should not add new bundle to any worksheet, or 0 otherwise. Default is 0. - `wait_for_upload`: 1 if the bundle state should be initialized to "uploading" regardless of the bundle type, or 0 otherwise. Used when copying bundles from another CodaLab instance, this prevents these new bundles from being executed by the BundleManager. Default is 0. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') after_sort_key = request.query.get('after_sort_key') detached = query_get_bool('detached', default=False) if worksheet_uuid is None: abort( http.client.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter", ) # Deserialize bundle fields bundles = (BundleSchema(strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load( request.json).data) # Check for all necessary permissions worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id metadata = bundle.get("metadata", {}) if metadata.get("link_url"): bundle['state'] = State.READY elif issubclass(bundle_class, UploadedBundle) or query_get_bool( 'wait_for_upload', False): bundle['state'] = State.UPLOADING else: bundle['state'] = State.CREATED bundle[ 'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) # Inherit worksheet permissions group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid) set_bundle_permissions([{ 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions]) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: local.model.add_worksheet_items( worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)], after_sort_key) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted # right after being created. bundles = [ bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict ] return BundleSchema(many=True).dump(bundles).data
def add_worksheet_item(self, worksheet_uuid, bundle_uuid): self.client.add_worksheet_item( worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))