def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False):
'''
Return information about the bundles.
get_children: whether we want to return information about the children too.
'''
if len(uuids) == 0:
return {}
bundles = self.model.batch_get_bundles(uuid=uuids)
bundle_dict = {bundle.uuid: self._bundle_to_bundle_info(bundle) for bundle in bundles}
# Check permissions after get the bundle information because some uuids
# might be invalid, and we don't want to check permissions because
# that's going to fail (for no good reason).
check_has_read_permission_on_bundles(self.model, self._current_user(), bundle_dict.keys())
# Lookup the user names of all the owners
ids = [info['owner_id'] for info in bundle_dict.values()]
users = self.auth_handler.get_users('ids', ids)
if users:
for info in bundle_dict.values():
user = users[info['owner_id']]
info['owner_name'] = user.name if user else None
info['owner'] = '%s(%s)' % (info['owner_name'], info['owner_id'])
if get_children:
result = self.model.get_children_uuids(uuids)
# Gather all children bundle uuids
children_uuids = [uuid for l in result.values() for uuid in l]
# Lookup bundle names
names = self.model.get_bundle_names(children_uuids)
# Fill in info
for uuid, info in bundle_dict.items():
info['children'] = [{'uuid': uuid, 'metadata': {'name': names[uuid]}} for uuid in result[uuid]]
if get_host_worksheets:
# bundle_uuids -> list of worksheet_uuids
result = self.model.get_host_worksheet_uuids(uuids)
# Gather all worksheet uuids
worksheet_uuids = [uuid for l in result.values() for uuid in l]
worksheets = dict((worksheet.uuid, worksheet) for worksheet in self.model.batch_get_worksheets(fetch_items=False, uuid=worksheet_uuids))
# Fill the info
for uuid, info in bundle_dict.items():
info['host_worksheets'] = [{'uuid': worksheet_uuid, 'name': worksheets[worksheet_uuid].name} for worksheet_uuid in result[uuid]]
return bundle_dict
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run):
'''
old_inputs: list of bundle uuids
old_output: bundle uuid that we produced
new_inputs: list of bundle uuids that are analogous to old_inputs
new_output_name: name of the bundle to create to be analogous to old_output (possibly None)
worksheet_uuid: add newly created bundles to this worksheet
depth: how far to do a BFS up
shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets.
'''
#print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name)
# Return the worksheet items that occur right before the given bundle_uuid
worksheet_cache = {} # worksheet_uuid => items
def add_with_prelude_items(old_bundle_uuid, new_bundle_uuid):
'''
Add new_bundle_uuid to the current worksheet (along with the occurrences on the worksheet).
Also add the prelude (items that occur right before old_bundle_uuid on a worksheet).
Note that new_bundle_uuid might get multiple times.
'''
host_worksheet_uuids = self.model.get_host_worksheet_uuids([old_bundle_uuid])[old_bundle_uuid]
if len(host_worksheet_uuids) > 0:
# Choose a single worksheet.
if worksheet_uuid in host_worksheet_uuids:
# If current worksheet is one of them, favor that one.
host_worksheet_uuid = worksheet_uuid
else:
# Choose an arbitrary one
host_worksheet_uuid = host_worksheet_uuids[0]
if host_worksheet_uuid in worksheet_cache:
worksheet_info = worksheet_cache[host_worksheet_uuid]
else:
worksheet_info = worksheet_cache[host_worksheet_uuid] = \
self.get_worksheet_info(host_worksheet_uuid, fetch_items=True)
# Look for items that appear right before the old_bundle_uuid
collect_items = []
for item in worksheet_info['items']:
(bundle_info, subworkheet_info, value_obj, type) = item
if type == worksheet_util.TYPE_BUNDLE and bundle_info['uuid'] == old_bundle_uuid:
# Ended in the target old_bundle_uuid, flush the prelude gathered so far.
for item2 in collect_items:
self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2))
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
collect_items = []
elif type == worksheet_util.TYPE_MARKUP and value_obj == '':
collect_items = []
elif type == worksheet_util.TYPE_BUNDLE:
collect_items = []
else:
collect_items.append(item)
# Build the graph (get all the infos).
# If old_output is given, look at ancestors of old_output until we
# reached some depth. If it's not given, we first get all the
# descendants first, and then get their ancestors.
infos = {} # uuid -> bundle info
if old_output:
bundle_uuids = [old_output]
else:
bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth)
all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order
for _ in range(depth):
new_bundle_uuids = []
for bundle_uuid in bundle_uuids:
if bundle_uuid in infos: continue # Already visited
info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid)
for dep in info['dependencies']:
parent_uuid = dep['parent_uuid']
if parent_uuid not in infos:
new_bundle_uuids.append(parent_uuid)
all_bundle_uuids = new_bundle_uuids + all_bundle_uuids
bundle_uuids = new_bundle_uuids
# Make sure we have read access to all the bundles involved here.
# TODO: need to double check that this is right.
check_has_read_permission_on_bundles(self.model, self._current_user(), list(infos.keys()))
# Now go recursively create the bundles.
old_to_new = {} # old_uuid -> new_uuid
downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid)
plan = [] # sequence of (old, new) bundle infos to make
for old, new in zip(old_inputs, new_inputs):
old_to_new[old] = new
downstream.add(old)
# Return corresponding new_bundle_uuid
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
#print old_bundle_uuid, 'cached'
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
#print old_bundle_uuid, 'no information'
return old_bundle_uuid
# Get information about the old bundle.
info = infos[old_bundle_uuid]
new_dependencies = [{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test
'child_path': dep['child_path']
} for dep in info['dependencies']]
# We're downstream, so need to make a new bundle
if any(dep['parent_uuid'] in downstream for dep in info['dependencies']):
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
old_bundle_name = info['metadata']['name']
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
new_bundle_uuid = self.derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, None)
# Add to worksheet
if shadow:
self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid)
else:
# Copy the markup and directives right before the old
add_with_prelude_items(old_bundle_uuid, new_bundle_uuid)
new_info['uuid'] = new_bundle_uuid
plan.append((info, new_info))
downstream.add(old_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
# Put initial newline to delimit things
if not dry_run and not shadow:
self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item(''))
if old_output:
recurse(old_output)
else:
# Don't have a particular output we're targetting, so just create
# new versions of all the uuids.
for uuid in all_bundle_uuids: recurse(uuid)
return plan
def download_target(self, target, follow_symlinks):
check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]])
# Don't need to download anything because it's already local.
# Note that we can't really enforce follow_symlinks, but this is okay,
# because we will follow them when we copy it from the target path.
return (self.get_target_path(target), None)
def open_target_handle(self, target):
check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
return open(path) if path and os.path.exists(path) else None
def head_target(self, target, num_lines):
check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
return path_util.read_lines(path, num_lines)
def cat_target(self, target, out):
check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
path_util.cat(path, out)
def get_target_info(self, target, depth):
check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
return path_util.get_info(path, depth)
def open_target(self, target):
check_has_read_permission_on_bundles(self.model, self._current_user(), [target[0]])
path = self.get_target_path(target)
path_util.check_isfile(path, 'open_target')
return open(path)
