def install_dependencies(self, bundle_store, parent_dict, path, rel): ''' Symlink this bundle's dependencies into the directory at path. The caller is responsible for cleaning up this directory. ''' precondition(os.path.isabs(path), '%s is a relative path!' % (path,)) for dep in self.dependencies: parent = parent_dict[dep.parent_uuid] # Compute an absolute target and check that the dependency exists. target = path_util.safe_join( bundle_store.get_location(parent.data_hash), dep.parent_path, ) if not os.path.exists(target): parent_spec = getattr(parent.metadata, 'name', parent.uuid) target_text = path_util.safe_join(parent_spec, dep.parent_path) raise UsageError('Target not found: %s' % (target_text,)) if rel: # Create a symlink that points to the dependency's relative target. target = path_util.safe_join( (os.pardir if dep.child_path else ''), bundle_store.get_location(parent.data_hash, relative=True), dep.parent_path, ) link_path = path_util.safe_join(path, dep.child_path) os.symlink(target, link_path)
def format_basic_info(self, info): metadata = collections.defaultdict(lambda: None, info['metadata']) # Format some simple fields of the basic info string. fields = { 'bundle_type': info['bundle_type'], 'uuid': info['uuid'], 'data_hash': info['data_hash'] or '<no hash>', 'state': info['state'], 'name': metadata['name'] or '<no name>', 'description': metadata['description'] or '<no description>', } # Format statistics about this bundle - creation time, runtime, size, etc. stats = [] if 'created' in metadata: stats.append('Created: %s' % (self.time_str(metadata['created']),)) if 'data_size' in metadata: stats.append('Size: %s' % (self.size_str(metadata['data_size']),)) fields['stats'] = 'Stats:\n %s\n' % ('\n '.join(stats),) if stats else '' # Compute a nicely-formatted list of hard dependencies. Since this type of # dependency is realized within this bundle as a symlink to another bundle, # label these dependencies as 'references' in the UI. fields['hard_dependencies'] = '' if info['hard_dependencies']: deps = info['hard_dependencies'] if len(deps) == 1 and not deps[0]['child_path']: fields['hard_dependencies'] = 'Reference:\n %s\n' % ( path_util.safe_join(deps[0]['parent_uuid'], deps[0]['parent_path']),) else: fields['hard_dependencies'] = 'References:\n%s\n' % ('\n'.join( ' %s:%s' % ( dep['child_path'], path_util.safe_join(dep['parent_uuid'], dep['parent_path']), ) for dep in sorted(deps, key=lambda dep: dep['child_path']) )) # Compute a nicely-formatted failure message, if this bundle failed. # It is possible for bundles that are not failed to have failure messages: # for example, if a bundle is killed in the database after running for too # long then succeeds afterwards, it will be in this state. fields['failure_message'] = '' if info['state'] == State.FAILED and metadata['failure_message']: fields['failure_message'] = 'Failure message:\n %s\n' % ('\n '.join( metadata['failure_message'].split('\n') )) # Return the formatted summary of the bundle info. return ''' {bundle_type}: {name} {description} UUID: {uuid} Hash: {data_hash} State: {state} {stats}{hard_dependencies}{failure_message} '''.format(**fields).strip()
def _make_bundle(self, bundle): try: bundle_location = self._bundle_store.get_bundle_location( bundle.uuid) path = os.path.normpath(bundle_location) deps = [] for dep in bundle.dependencies: parent_bundle_path = os.path.normpath( self._bundle_store.get_bundle_location(dep.parent_uuid)) dependency_path = os.path.normpath( os.path.join(parent_bundle_path, dep.parent_path)) if not dependency_path.startswith(parent_bundle_path) or ( not os.path.islink(dependency_path) and not os.path.exists(dependency_path)): raise Exception('Invalid dependency %s' % (path_util.safe_join( dep.parent_uuid, dep.parent_path))) child_path = os.path.normpath( os.path.join(path, dep.child_path)) if not child_path.startswith(path): raise Exception('Invalid key for dependency: %s' % (dep.child_path)) deps.append((dependency_path, child_path)) remove_path(path) if len(deps) == 1 and deps[0][1] == path: path_util.copy(deps[0][0], path, follow_symlinks=False) else: os.mkdir(path) for dependency_path, child_path in deps: path_util.copy(dependency_path, child_path, follow_symlinks=False) self._model.update_disk_metadata(bundle, bundle_location, enforce_disk_quota=True) logger.info('Finished making bundle %s', bundle.uuid) self._model.update_bundle(bundle, {'state': State.READY}) except Exception as e: logger.info('Failing bundle %s: %s', bundle.uuid, str(e)) self._model.update_bundle(bundle, { 'state': State.FAILED, 'metadata': { 'failure_message': str(e) } }) finally: with self._make_uuids_lock: self._make_uuids.remove(bundle.uuid)
def process_dep(dep): parent = parent_dict[dep.parent_uuid] # Compute an absolute target and check that the dependency exists. if not parent.data_hash: raise UsageError("Parent %s does not have data hash" % parent) target = path_util.safe_join(bundle_store.get_location(parent.data_hash), dep.parent_path) if not os.path.exists(target): parent_spec = getattr(parent.metadata, "name", parent.uuid) target_text = path_util.safe_join(parent_spec, dep.parent_path) raise UsageError("Target not found: %s" % (target_text,)) if relative_symlinks: # Create a symlink that points to the dependency's relative target. target = path_util.safe_join( (os.pardir if dep.child_path else ""), bundle_store.get_location(parent.data_hash, relative=True), dep.parent_path, ) link_path = path_util.safe_join(dest_path, dep.child_path) return (target, link_path)
def process_dep(dep): parent = parent_dict[dep.parent_uuid] # Compute an absolute target and check that the dependency exists. if not parent.data_hash: raise UsageError('Parent %s does not have data hash' % parent) target = path_util.safe_join( bundle_store.get_location(parent.data_hash), dep.parent_path, ) if not os.path.exists(target): parent_spec = getattr(parent.metadata, 'name', parent.uuid) target_text = path_util.safe_join(parent_spec, dep.parent_path) raise UsageError('Target not found: %s' % (target_text, )) if relative_symlinks: # Create a symlink that points to the dependency's relative target. target = path_util.safe_join( (os.pardir if dep.child_path else ''), bundle_store.get_location(parent.data_hash, relative=True), dep.parent_path, ) link_path = path_util.safe_join(dest_path, dep.child_path) return (target, link_path)
def get_target_path(bundle_store, model, target): """ Return the on-disk location of the target (bundle_uuid, subpath) pair. """ (uuid, path) = target bundle = model.get_bundle(uuid) if not bundle.data_hash: # Note that the bundle might not be ready, but return the location anyway to the temporary directory. bundle_root = get_current_location(bundle_store, uuid) else: bundle_root = bundle_store.get_location(bundle.data_hash) final_path = path_util.safe_join(bundle_root, path) result = path_util.TargetPath(final_path, target) return result
def _make_bundle(self, bundle): try: path = os.path.normpath(self._bundle_store.get_bundle_location(bundle.uuid)) deps = [] for dep in bundle.dependencies: parent_bundle_path = os.path.normpath( self._bundle_store.get_bundle_location(dep.parent_uuid) ) dependency_path = os.path.normpath( os.path.join(parent_bundle_path, dep.parent_path) ) if not dependency_path.startswith(parent_bundle_path) or ( not os.path.islink(dependency_path) and not os.path.exists(dependency_path) ): raise Exception( 'Invalid dependency %s' % (path_util.safe_join(dep.parent_uuid, dep.parent_path)) ) child_path = os.path.normpath(os.path.join(path, dep.child_path)) if not child_path.startswith(path): raise Exception('Invalid key for dependency: %s' % (dep.child_path)) deps.append((dependency_path, child_path)) remove_path(path) if len(deps) == 1 and deps[0][1] == path: path_util.copy(deps[0][0], path, follow_symlinks=False) else: os.mkdir(path) for dependency_path, child_path in deps: path_util.copy(dependency_path, child_path, follow_symlinks=False) self._upload_manager.update_metadata_and_save(bundle, enforce_disk_quota=True) logger.info('Finished making bundle %s', bundle.uuid) self._model.update_bundle(bundle, {'state': State.READY}) except Exception as e: logger.info('Failing bundle %s: %s', bundle.uuid, str(e)) self._model.update_bundle( bundle, {'state': State.FAILED, 'metadata': {'failure_message': str(e)}} ) finally: with self._make_uuids_lock: self._make_uuids.remove(bundle.uuid)
def get_target_path(bundle_store, model, target): ''' Return the on-disk location of the target (bundle_spec, path) pair. ''' (bundle_spec, path) = target uuid = get_spec_uuid(model, bundle_spec) bundle = model.get_bundle(uuid) if not bundle.data_hash: message = 'Unexpected: %s is ready but it has no data hash!' % (bundle,) precondition(bundle.state != State.READY, message) if bundle.state == State.FAILED: raise UsageError('%s failed unrecoverably' % (bundle,)) else: raise UsageError('%s has not yet been executed' % (bundle,)) bundle_root = bundle_store.get_location(bundle.data_hash) final_path = path_util.safe_join(bundle_root, path) result = path_util.TargetPath(final_path) result.target = target return result
def get_target_path(bundle_store, model, target): ''' Return the on-disk location of the target (bundle_uuid, subpath) pair. ''' (uuid, path) = target bundle = model.get_bundle(uuid) if not bundle.data_hash: # Note that the bundle might not be done, but return the location anyway to the temporary directory bundle_root = get_current_location(bundle_store, uuid) else: bundle_root = bundle_store.get_location(bundle.data_hash) final_path = path_util.safe_join(bundle_root, path) # This is too restrictive because it means we can't follow any of the # components of a make bundle. #path_util.check_under_path(final_path, bundle_root) result = path_util.TargetPath(final_path) result.target = target return result
def _make_bundle(self, bundle): try: bundle_link_url = getattr(bundle.metadata, "link_url", None) bundle_location = bundle_link_url or self._bundle_store.get_bundle_location( bundle.uuid) path = os.path.normpath(bundle_location) deps = [] parent_bundle_link_urls = self._model.get_bundle_metadata( [dep.parent_uuid for dep in bundle.dependencies], "link_url") for dep in bundle.dependencies: parent_bundle_link_url = parent_bundle_link_urls.get( dep.parent_uuid) try: parent_bundle_path = parent_bundle_link_url or os.path.normpath( self._bundle_store.get_bundle_location( dep.parent_uuid)) except NotFoundError: raise Exception('Invalid dependency %s' % (path_util.safe_join( dep.parent_uuid, dep.parent_path))) # TODO(Ashwin): make this logic non-fs specific. dependency_path = os.path.normpath( os.path.join(parent_bundle_path, dep.parent_path)) if not dependency_path.startswith(parent_bundle_path) or ( not os.path.islink(dependency_path) and not os.path.exists(dependency_path)): raise Exception('Invalid dependency %s' % (path_util.safe_join( dep.parent_uuid, dep.parent_path))) child_path = os.path.normpath( os.path.join(path, dep.child_path)) if not child_path.startswith(path): raise Exception('Invalid key for dependency: %s' % (dep.child_path)) deps.append((dependency_path, child_path)) remove_path(path) if len(deps) == 1 and deps[0][1] == path: path_util.copy(deps[0][0], path, follow_symlinks=False) else: os.mkdir(path) for dependency_path, child_path in deps: path_util.copy(dependency_path, child_path, follow_symlinks=False) # TODO(Ashwin): fix self._model.update_disk_metadata(bundle, bundle_location, enforce_disk_quota=True) logger.info('Finished making bundle %s', bundle.uuid) self._model.update_bundle(bundle, {'state': State.READY}) except Exception as e: logger.info('Failing bundle %s: %s', bundle.uuid, str(e)) self._model.update_bundle( bundle, { 'state': State.FAILED, 'metadata': { 'failure_message': str(e), 'error_traceback': traceback.format_exc(), }, }, ) finally: with self._make_uuids_lock: self._make_uuids.remove(bundle.uuid)