def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False): ''' Return information about the bundles. get_children: whether we want to return information about the children too. ''' if len(uuids) == 0: return {} bundles = self.model.batch_get_bundles(uuid=uuids) bundle_dict = {bundle.uuid: self._bundle_to_bundle_info(bundle) for bundle in bundles} # Check permissions after get the bundle information because some uuids # might be invalid, and we don't want to check permissions because # that's going to fail (for no good reason). check_has_read_permission_on_bundles(self.model, self._current_user(), bundle_dict.keys()) # Lookup the user names of all the owners ids = [info['owner_id'] for info in bundle_dict.values()] users = self.auth_handler.get_users('ids', ids) if users: for info in bundle_dict.values(): user = users[info['owner_id']] info['owner_name'] = user.name if user else None info['owner'] = '%s(%s)' % (info['owner_name'], info['owner_id']) if get_children: result = self.model.get_children_uuids(uuids) # Gather all children bundle uuids children_uuids = [uuid for l in result.values() for uuid in l] # Lookup bundle names names = self.model.get_bundle_names(children_uuids) # Fill in info for uuid, info in bundle_dict.items(): info['children'] = [{'uuid': uuid, 'metadata': {'name': names[uuid]}} for uuid in result[uuid]] if get_host_worksheets: # bundle_uuids -> list of worksheet_uuids result = self.model.get_host_worksheet_uuids(uuids) # Gather all worksheet uuids worksheet_uuids = [uuid for l in result.values() for uuid in l] worksheets = dict((worksheet.uuid, worksheet) for worksheet in self.model.batch_get_worksheets(fetch_items=False, uuid=worksheet_uuids)) # Fill the info for uuid, info in bundle_dict.items(): info['host_worksheets'] = [{'uuid': worksheet_uuid, 'name': worksheets[worksheet_uuid].name} for worksheet_uuid in result[uuid]] return bundle_dict
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run): ''' old_inputs: list of bundle uuids old_output: bundle uuid that we produced new_inputs: list of bundle uuids that are analogous to old_inputs new_output_name: name of the bundle to create to be analogous to old_output (possibly None) worksheet_uuid: add newly created bundles to this worksheet depth: how far to do a BFS up shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets. ''' #print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name) # Return the worksheet items that occur right before the given bundle_uuid worksheet_cache = {} # worksheet_uuid => items def add_with_prelude_items(old_bundle_uuid, new_bundle_uuid): ''' Add new_bundle_uuid to the current worksheet (along with the occurrences on the worksheet). Also add the prelude (items that occur right before old_bundle_uuid on a worksheet). Note that new_bundle_uuid might get multiple times. ''' host_worksheet_uuids = self.model.get_host_worksheet_uuids([old_bundle_uuid])[old_bundle_uuid] if len(host_worksheet_uuids) > 0: # Choose a single worksheet. if worksheet_uuid in host_worksheet_uuids: # If current worksheet is one of them, favor that one. host_worksheet_uuid = worksheet_uuid else: # Choose an arbitrary one host_worksheet_uuid = host_worksheet_uuids[0] if host_worksheet_uuid in worksheet_cache: worksheet_info = worksheet_cache[host_worksheet_uuid] else: worksheet_info = worksheet_cache[host_worksheet_uuid] = \ self.get_worksheet_info(host_worksheet_uuid, fetch_items=True) # Look for items that appear right before the old_bundle_uuid collect_items = [] for item in worksheet_info['items']: (bundle_info, subworkheet_info, value_obj, type) = item if type == worksheet_util.TYPE_BUNDLE and bundle_info['uuid'] == old_bundle_uuid: # Ended in the target old_bundle_uuid, flush the prelude gathered so far. for item2 in collect_items: self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2)) self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid)) collect_items = [] elif type == worksheet_util.TYPE_MARKUP and value_obj == '': collect_items = [] elif type == worksheet_util.TYPE_BUNDLE: collect_items = [] else: collect_items.append(item) # Build the graph (get all the infos). # If old_output is given, look at ancestors of old_output until we # reached some depth. If it's not given, we first get all the # descendants first, and then get their ancestors. infos = {} # uuid -> bundle info if old_output: bundle_uuids = [old_output] else: bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth) all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order for _ in range(depth): new_bundle_uuids = [] for bundle_uuid in bundle_uuids: if bundle_uuid in infos: continue # Already visited info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid) for dep in info['dependencies']: parent_uuid = dep['parent_uuid'] if parent_uuid not in infos: new_bundle_uuids.append(parent_uuid) all_bundle_uuids = new_bundle_uuids + all_bundle_uuids bundle_uuids = new_bundle_uuids # Make sure we have read access to all the bundles involved here. # TODO: need to double check that this is right. check_has_read_permission_on_bundles(self.model, self._current_user(), list(infos.keys())) # Now go recursively create the bundles. old_to_new = {} # old_uuid -> new_uuid downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid) plan = [] # sequence of (old, new) bundle infos to make for old, new in zip(old_inputs, new_inputs): old_to_new[old] = new downstream.add(old) # Return corresponding new_bundle_uuid def recurse(old_bundle_uuid): if old_bundle_uuid in old_to_new: #print old_bundle_uuid, 'cached' return old_to_new[old_bundle_uuid] # Don't have any more information (because we probably hit the maximum depth) if old_bundle_uuid not in infos: #print old_bundle_uuid, 'no information' return old_bundle_uuid # Get information about the old bundle. info = infos[old_bundle_uuid] new_dependencies = [{ 'parent_uuid': recurse(dep['parent_uuid']), 'parent_path': dep['parent_path'], 'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test 'child_path': dep['child_path'] } for dep in info['dependencies']] # We're downstream, so need to make a new bundle if any(dep['parent_uuid'] in downstream for dep in info['dependencies']): # Now create a new bundle that mimics the old bundle. # Only change the name if the output name is supplied. old_bundle_name = info['metadata']['name'] new_info = copy.deepcopy(info) new_metadata = new_info['metadata'] if new_output_name: if old_bundle_uuid == old_output: new_metadata['name'] = new_output_name else: # Just make up a name heuristically new_metadata['name'] = new_output_name + '-' + info['metadata']['name'] # Remove all the automatically generated keys cls = get_bundle_subclass(new_info['bundle_type']) for spec in cls.METADATA_SPECS: if spec.generated and spec.key in new_metadata: new_metadata.pop(spec.key) # Set the targets targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies] if dry_run: new_bundle_uuid = None else: new_bundle_uuid = self.derive_bundle(new_info['bundle_type'], \ targets, new_info['command'], new_metadata, None) # Add to worksheet if shadow: self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid) else: # Copy the markup and directives right before the old add_with_prelude_items(old_bundle_uuid, new_bundle_uuid) new_info['uuid'] = new_bundle_uuid plan.append((info, new_info)) downstream.add(old_bundle_uuid) else: new_bundle_uuid = old_bundle_uuid old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it return new_bundle_uuid # Put initial newline to delimit things if not dry_run and not shadow: self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item('')) if old_output: recurse(old_output) else: # Don't have a particular output we're targetting, so just create # new versions of all the uuids. for uuid in all_bundle_uuids: recurse(uuid) return plan
def download_target(self, target, follow_symlinks): check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]]) # Don't need to download anything because it's already local. # Note that we can't really enforce follow_symlinks, but this is okay, # because we will follow them when we copy it from the target path. return (self.get_target_path(target), None)
def open_target_handle(self, target): check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) return open(path) if path and os.path.exists(path) else None
def head_target(self, target, num_lines): check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) return path_util.read_lines(path, num_lines)
def cat_target(self, target, out): check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) path_util.cat(path, out)
def get_target_info(self, target, depth): check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) return path_util.get_info(path, depth)
def open_target(self, target): check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]]) path = self.get_target_path(target) path_util.check_isfile(path, 'open_target') return open(path)