def replace_items(worksheet_uuid): """ Replace worksheet items with 'ids' with new 'items'. The new items will be inserted after the after_sort_key """ worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) ids = request.json.get('ids', []) item_type = request.json.get('item_type', 'markup') after_sort_key = request.json.get('after_sort_key') # Default to process only markup items. if item_type == "markup": items = [ worksheet_util.markup_item(item) for item in request.json.get('items', []) ] elif item_type == "bundle": items = [ worksheet_util.bundle_item(item) for item in request.json.get('items', []) ] elif item_type == "directive": # Note: for directives, the item should not include the preceding "%" symbol items = [ worksheet_util.directive_item(item) for item in request.json.get('items', []) ] local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def interpret_search(query): """ Input: specification of a bundle search query. Output: worksheet items based on the result of issuing the search query. """ # Perform search keywords = rest_util.resolve_owner_in_keywords(query['keywords']) bundle_uuids = local.model.search_bundle_uuids(request.user.user_id, keywords) # Single number, just print it out... if not isinstance(bundle_uuids, list): return interpret_items(query['schemas'], [markup_item(str(bundle_uuids))]) # Set display items = [directive_item(('display', ) + tuple(query['display']))] # Show bundles bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: items.append(bundle_item(bundle_infos[bundle_uuid])) # Finally, interpret the items return interpret_items(query['schemas'], items)
def expand_raw_item(raw_item): """ Raw items that include searches must be expanded into more raw items. Input: Raw item. Output: Array of raw items. If raw item does not need expanding, this returns an 1-length array that contains original raw item, otherwise it contains the search result. You do not need to call resolve_items_into_infos on the returned raw_items. """ (bundle_info, subworksheet_info, value_obj, item_type, id, sort_key) = raw_item is_search = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'search' is_wsearch = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'wsearch' if is_search or is_wsearch: command = get_command(value_obj) keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) if search_result['is_aggregate']: # Add None's for the 'id' and 'sort_key' of the tuple, since these # items are not really worksheet items. raw_items.append( markup_item(str(search_result['result'])) + (None, None)) else: bundle_uuids = search_result['result'] bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: # Since bundle UUID's are queried first, we can't assume a UUID exists in # the subsequent bundle info query. if bundle_uuid in bundle_infos: raw_items.append( bundle_item(bundle_infos[bundle_uuid]) + (None, None)) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append( subworksheet_item(worksheet_info) + (None, None)) return raw_items else: return [raw_item]
def replace_items(worksheet_uuid): """ Replace worksheet items with 'ids' with new 'items'. The new items will be inserted after the after_sort_key """ worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) ids = request.json.get('ids', []) after_sort_key = request.json.get('after_sort_key') # Default to process only markup items. items = [ worksheet_util.markup_item(item) for item in request.json.get('items', []) ] local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def expand_raw_item(raw_item): """ Raw items that include searches must be expanded into more raw items. Input: Raw item. Output: Array of raw items. If raw item does not need expanding, this returns an 1-length array that contains original raw item, otherwise it contains the search result. You do not need to call resolve_items_into_infos on the returned raw_items. """ (bundle_info, subworksheet_info, value_obj, item_type) = raw_item is_search = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'search' is_wsearch = item_type == TYPE_DIRECTIVE and get_command( value_obj) == 'wsearch' if is_search or is_wsearch: command = get_command(value_obj) keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundle_uuids( request.user.user_id, keywords) if not isinstance(search_result, list): raw_items.append(markup_item(str(search_result))) else: bundle_uuids = search_result bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: raw_items.append(bundle_item(bundle_infos[bundle_uuid])) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append(subworksheet_item(worksheet_info)) return raw_items else: return [raw_item]
def perform_search_query(value_obj): """ Perform a search query and return the resulting raw items. Input: directive that is tokenized by formatting.string_to_tokens(), such as formatting.string_to_tokens("search 0x .limit=100") """ command = get_command(value_obj) is_search = command == 'search' is_wsearch = command == 'wsearch' if is_search or is_wsearch: keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) if search_result['is_aggregate']: # Add None's for the 'id' and 'sort_key' of the tuple, since these # items are not really worksheet items. raw_items.append( markup_item(str(search_result['result'])) + (None, None)) else: bundle_uuids = search_result['result'] bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: # Since bundle UUID's are queried first, we can't assume a UUID exists in # the subsequent bundle info query. if bundle_uuid in bundle_infos: raw_items.append( bundle_item(bundle_infos[bundle_uuid]) + (None, None)) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append( subworksheet_item(worksheet_info) + (None, None)) return raw_items else: # Not a search query return []
def expand_raw_item(raw_item): """ Raw items that include searches must be expanded into more raw items. Input: Raw item. Output: Array of raw items. If raw item does not need expanding, this returns an 1-length array that contains original raw item, otherwise it contains the search result. You do not need to call resolve_items_into_infos on the returned raw_items. """ (bundle_info, subworksheet_info, value_obj, item_type) = raw_item is_search = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'search' is_wsearch = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'wsearch' if is_search or is_wsearch: command = get_command(value_obj) keywords = value_obj[1:] raw_items = [] if is_search: keywords = rest_util.resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) if search_result['is_aggregate']: raw_items.append(markup_item(str(search_result['result']))) else: bundle_uuids = search_result['result'] bundle_infos = rest_util.get_bundle_infos(bundle_uuids) for bundle_uuid in bundle_uuids: raw_items.append(bundle_item(bundle_infos[bundle_uuid])) elif is_wsearch: worksheet_infos = search_worksheets(keywords) for worksheet_info in worksheet_infos: raw_items.append(subworksheet_item(worksheet_info)) return raw_items else: return [raw_item]
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run): ''' old_inputs: list of bundle uuids old_output: bundle uuid that we produced new_inputs: list of bundle uuids that are analogous to old_inputs new_output_name: name of the bundle to create to be analogous to old_output (possibly None) worksheet_uuid: add newly created bundles to this worksheet depth: how far to do a BFS up shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets. ''' #print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name) # Return the worksheet items that occur right before the given bundle_uuid worksheet_cache = {} # worksheet_uuid => items def add_with_prelude_items(old_bundle_uuid, new_bundle_uuid): ''' Add new_bundle_uuid to the current worksheet (along with the occurrences on the worksheet). Also add the prelude (items that occur right before old_bundle_uuid on a worksheet). Note that new_bundle_uuid might get multiple times. ''' host_worksheet_uuids = self.model.get_host_worksheet_uuids([old_bundle_uuid])[old_bundle_uuid] if len(host_worksheet_uuids) > 0: # Choose a single worksheet. if worksheet_uuid in host_worksheet_uuids: # If current worksheet is one of them, favor that one. host_worksheet_uuid = worksheet_uuid else: # Choose an arbitrary one host_worksheet_uuid = host_worksheet_uuids[0] if host_worksheet_uuid in worksheet_cache: worksheet_info = worksheet_cache[host_worksheet_uuid] else: worksheet_info = worksheet_cache[host_worksheet_uuid] = \ self.get_worksheet_info(host_worksheet_uuid, fetch_items=True) # Look for items that appear right before the old_bundle_uuid collect_items = [] for item in worksheet_info['items']: (bundle_info, subworkheet_info, value_obj, type) = item if type == worksheet_util.TYPE_BUNDLE and bundle_info['uuid'] == old_bundle_uuid: # Ended in the target old_bundle_uuid, flush the prelude gathered so far. for item2 in collect_items: self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2)) self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) collect_items = [] elif type == worksheet_util.TYPE_MARKUP and value_obj == '': collect_items = [] elif type == worksheet_util.TYPE_BUNDLE: collect_items = [] else: collect_items.append(item) # Build the graph (get all the infos). # If old_output is given, look at ancestors of old_output until we # reached some depth. If it's not given, we first get all the # descendants first, and then get their ancestors. infos = {} # uuid -> bundle info if old_output: bundle_uuids = [old_output] else: bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth) all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order for _ in range(depth): new_bundle_uuids = [] for bundle_uuid in bundle_uuids: if bundle_uuid in infos: continue # Already visited info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid) for dep in info['dependencies']: parent_uuid = dep['parent_uuid'] if parent_uuid not in infos: new_bundle_uuids.append(parent_uuid) all_bundle_uuids = new_bundle_uuids + all_bundle_uuids bundle_uuids = new_bundle_uuids # Make sure we have read access to all the bundles involved here. # TODO: need to double check that this is right. check_has_read_permission_on_bundles(self.model, self._current_user(), list(infos.keys())) # Now go recursively create the bundles. old_to_new = {} # old_uuid -> new_uuid downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid) plan = [] # sequence of (old, new) bundle infos to make for old, new in zip(old_inputs, new_inputs): old_to_new[old] = new downstream.add(old) # Return corresponding new_bundle_uuid def recurse(old_bundle_uuid): if old_bundle_uuid in old_to_new: #print old_bundle_uuid, 'cached' return old_to_new[old_bundle_uuid] # Don't have any more information (because we probably hit the maximum depth) if old_bundle_uuid not in infos: #print old_bundle_uuid, 'no information' return old_bundle_uuid # Get information about the old bundle. info = infos[old_bundle_uuid] new_dependencies = [{ 'parent_uuid': recurse(dep['parent_uuid']), 'parent_path': dep['parent_path'], 'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test 'child_path': dep['child_path'] } for dep in info['dependencies']] # We're downstream, so need to make a new bundle if any(dep['parent_uuid'] in downstream for dep in info['dependencies']): # Now create a new bundle that mimics the old bundle. # Only change the name if the output name is supplied. old_bundle_name = info['metadata']['name'] new_info = copy.deepcopy(info) new_metadata = new_info['metadata'] if new_output_name: if old_bundle_uuid == old_output: new_metadata['name'] = new_output_name else: # Just make up a name heuristically new_metadata['name'] = new_output_name + '-' + info['metadata']['name'] # Remove all the automatically generated keys cls = get_bundle_subclass(new_info['bundle_type']) for spec in cls.METADATA_SPECS: if spec.generated and spec.key in new_metadata: new_metadata.pop(spec.key) # Set the targets targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies] if dry_run: new_bundle_uuid = None else: new_bundle_uuid = self.derive_bundle(new_info['bundle_type'], \ targets, new_info['command'], new_metadata, None) # Add to worksheet if shadow: self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid) else: # Copy the markup and directives right before the old add_with_prelude_items(old_bundle_uuid, new_bundle_uuid) new_info['uuid'] = new_bundle_uuid plan.append((info, new_info)) downstream.add(old_bundle_uuid) else: new_bundle_uuid = old_bundle_uuid old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it return new_bundle_uuid # Put initial newline to delimit things if not dry_run and not shadow: self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item('')) if old_output: recurse(old_output) else: # Don't have a particular output we're targetting, so just create # new versions of all the uuids. for uuid in all_bundle_uuids: recurse(uuid) return plan
def newline(): self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item(''))
def _create_bundles(): """ Bulk create bundles. Query parameters: - `worksheet`: UUID of the parent worksheet of the new bundle, add to this worksheet if not detached or shadowing another bundle. The new bundle also inherits permissions from this worksheet. - `shadow`: UUID of the bundle to "shadow" (the new bundle will be added as an item immediately after this bundle in its parent worksheet). - `detached`: 1 if should not add new bundle to any worksheet, so the bundle does not have a hosted worksheet. This is set to 1, for example, if the user is uploading their avatar as a bundle. or 0 otherwise. Default is 0. - `wait_for_upload`: 1 if the bundle state should be initialized to "uploading" regardless of the bundle type, or 0 otherwise. Used when copying bundles from another CodaLab instance, this prevents these new bundles from being executed by the BundleManager. Default is 0. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') after_sort_key = request.query.get('after_sort_key') detached = query_get_bool('detached', default=False) after_image = query_get_bool('after_image', default=False) if not detached and worksheet_uuid is None: abort( http.client.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter", ) # Deserialize bundle fields bundles = ( BundleSchema(strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS) .load(request.json) .data ) # Check for all necessary permissions if not detached: worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id metadata = bundle.get("metadata", {}) if metadata.get("link_url"): bundle['state'] = State.READY elif issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False): bundle['state'] = State.UPLOADING else: bundle['state'] = State.CREATED if not detached: bundle['is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity else: bundle['is_anonymous'] = False bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) if not detached: # Inherit worksheet permissions; else, only the user will have all permissions on the bundle group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid ) set_bundle_permissions( [ { 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions ] ) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: items = [worksheet_util.bundle_item(bundle_uuid)] # Add a blank line after the image block in the worksheet source to ensure it is a separate block if after_image: items.insert(0, worksheet_util.markup_item('')) local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted # right after being created. bundles = [bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict] return BundleSchema(many=True).dump(bundles).data