Exemplo n.º 1
0
 def install_dependencies(self, bundle_store, parent_dict, path, rel):
     '''
     Symlink this bundle's dependencies into the directory at path.
     The caller is responsible for cleaning up this directory.
     '''
     precondition(os.path.isabs(path), '%s is a relative path!' % (path,))
     for dep in self.dependencies:
         parent = parent_dict[dep.parent_uuid]
         # Compute an absolute target and check that the dependency exists.
         target = path_util.safe_join(
           bundle_store.get_location(parent.data_hash),
           dep.parent_path,
         )
         if not os.path.exists(target):
             parent_spec = getattr(parent.metadata, 'name', parent.uuid)
             target_text = path_util.safe_join(parent_spec, dep.parent_path)
             raise UsageError('Target not found: %s' % (target_text,))
         if rel:
             # Create a symlink that points to the dependency's relative target.
             target = path_util.safe_join(
               (os.pardir if dep.child_path else ''),
               bundle_store.get_location(parent.data_hash, relative=True),
               dep.parent_path,
             )
         link_path = path_util.safe_join(path, dep.child_path)
         os.symlink(target, link_path)
Exemplo n.º 2
0
 def format_basic_info(self, info):
     metadata = collections.defaultdict(lambda: None, info['metadata'])
     # Format some simple fields of the basic info string.
     fields = {
       'bundle_type': info['bundle_type'],
       'uuid': info['uuid'],
       'data_hash': info['data_hash'] or '<no hash>',
       'state': info['state'],
       'name': metadata['name'] or '<no name>',
       'description': metadata['description'] or '<no description>',
     }
     # Format statistics about this bundle - creation time, runtime, size, etc.
     stats = []
     if 'created' in metadata:
         stats.append('Created: %s' % (self.time_str(metadata['created']),))
     if 'data_size' in metadata:
         stats.append('Size:    %s' % (self.size_str(metadata['data_size']),))
     fields['stats'] = 'Stats:\n  %s\n' % ('\n  '.join(stats),) if stats else ''
     # Compute a nicely-formatted list of hard dependencies. Since this type of
     # dependency is realized within this bundle as a symlink to another bundle,
     # label these dependencies as 'references' in the UI.
     fields['hard_dependencies'] = ''
     if info['hard_dependencies']:
         deps = info['hard_dependencies']
         if len(deps) == 1 and not deps[0]['child_path']:
             fields['hard_dependencies'] = 'Reference:\n  %s\n' % (
               path_util.safe_join(deps[0]['parent_uuid'], deps[0]['parent_path']),)
         else:
             fields['hard_dependencies'] = 'References:\n%s\n' % ('\n'.join(
               '  %s:%s' % (
                 dep['child_path'],
                 path_util.safe_join(dep['parent_uuid'], dep['parent_path']),
               ) for dep in sorted(deps, key=lambda dep: dep['child_path'])
             ))
     # Compute a nicely-formatted failure message, if this bundle failed.
     # It is possible for bundles that are not failed to have failure messages:
     # for example, if a bundle is killed in the database after running for too
     # long then succeeds afterwards, it will be in this state.
     fields['failure_message'] = ''
     if info['state'] == State.FAILED and metadata['failure_message']:
         fields['failure_message'] = 'Failure message:\n  %s\n' % ('\n  '.join(
           metadata['failure_message'].split('\n')
         ))
     # Return the formatted summary of the bundle info.
     return '''
 {bundle_type}: {name}
 {description}
   UUID:  {uuid}
   Hash:  {data_hash}
   State: {state}
 {stats}{hard_dependencies}{failure_message}
     '''.format(**fields).strip()
Exemplo n.º 3
0
    def _make_bundle(self, bundle):
        try:
            bundle_location = self._bundle_store.get_bundle_location(
                bundle.uuid)
            path = os.path.normpath(bundle_location)

            deps = []
            for dep in bundle.dependencies:
                parent_bundle_path = os.path.normpath(
                    self._bundle_store.get_bundle_location(dep.parent_uuid))
                dependency_path = os.path.normpath(
                    os.path.join(parent_bundle_path, dep.parent_path))
                if not dependency_path.startswith(parent_bundle_path) or (
                        not os.path.islink(dependency_path)
                        and not os.path.exists(dependency_path)):
                    raise Exception('Invalid dependency %s' %
                                    (path_util.safe_join(
                                        dep.parent_uuid, dep.parent_path)))

                child_path = os.path.normpath(
                    os.path.join(path, dep.child_path))
                if not child_path.startswith(path):
                    raise Exception('Invalid key for dependency: %s' %
                                    (dep.child_path))

                deps.append((dependency_path, child_path))

            remove_path(path)

            if len(deps) == 1 and deps[0][1] == path:
                path_util.copy(deps[0][0], path, follow_symlinks=False)
            else:
                os.mkdir(path)
                for dependency_path, child_path in deps:
                    path_util.copy(dependency_path,
                                   child_path,
                                   follow_symlinks=False)

            self._model.update_disk_metadata(bundle,
                                             bundle_location,
                                             enforce_disk_quota=True)
            logger.info('Finished making bundle %s', bundle.uuid)
            self._model.update_bundle(bundle, {'state': State.READY})
        except Exception as e:
            logger.info('Failing bundle %s: %s', bundle.uuid, str(e))
            self._model.update_bundle(bundle, {
                'state': State.FAILED,
                'metadata': {
                    'failure_message': str(e)
                }
            })
        finally:
            with self._make_uuids_lock:
                self._make_uuids.remove(bundle.uuid)
Exemplo n.º 4
0
        def process_dep(dep):
            parent = parent_dict[dep.parent_uuid]
            # Compute an absolute target and check that the dependency exists.
            if not parent.data_hash:
                raise UsageError("Parent %s does not have data hash" % parent)
            target = path_util.safe_join(bundle_store.get_location(parent.data_hash), dep.parent_path)
            if not os.path.exists(target):
                parent_spec = getattr(parent.metadata, "name", parent.uuid)
                target_text = path_util.safe_join(parent_spec, dep.parent_path)
                raise UsageError("Target not found: %s" % (target_text,))
            if relative_symlinks:
                # Create a symlink that points to the dependency's relative target.
                target = path_util.safe_join(
                    (os.pardir if dep.child_path else ""),
                    bundle_store.get_location(parent.data_hash, relative=True),
                    dep.parent_path,
                )
            link_path = path_util.safe_join(dest_path, dep.child_path)

            return (target, link_path)
Exemplo n.º 5
0
        def process_dep(dep):
            parent = parent_dict[dep.parent_uuid]
            # Compute an absolute target and check that the dependency exists.
            if not parent.data_hash:
                raise UsageError('Parent %s does not have data hash' % parent)
            target = path_util.safe_join(
                bundle_store.get_location(parent.data_hash),
                dep.parent_path,
            )
            if not os.path.exists(target):
                parent_spec = getattr(parent.metadata, 'name', parent.uuid)
                target_text = path_util.safe_join(parent_spec, dep.parent_path)
                raise UsageError('Target not found: %s' % (target_text, ))
            if relative_symlinks:
                # Create a symlink that points to the dependency's relative target.
                target = path_util.safe_join(
                    (os.pardir if dep.child_path else ''),
                    bundle_store.get_location(parent.data_hash, relative=True),
                    dep.parent_path,
                )
            link_path = path_util.safe_join(dest_path, dep.child_path)

            return (target, link_path)
Exemplo n.º 6
0
def get_target_path(bundle_store, model, target):
    """
    Return the on-disk location of the target (bundle_uuid, subpath) pair.
    """
    (uuid, path) = target
    bundle = model.get_bundle(uuid)
    if not bundle.data_hash:
        # Note that the bundle might not be ready, but return the location anyway to the temporary directory.
        bundle_root = get_current_location(bundle_store, uuid)
    else:
        bundle_root = bundle_store.get_location(bundle.data_hash)
    final_path = path_util.safe_join(bundle_root, path)

    result = path_util.TargetPath(final_path, target)
    return result
Exemplo n.º 7
0
    def _make_bundle(self, bundle):
        try:
            path = os.path.normpath(self._bundle_store.get_bundle_location(bundle.uuid))

            deps = []
            for dep in bundle.dependencies:
                parent_bundle_path = os.path.normpath(
                    self._bundle_store.get_bundle_location(dep.parent_uuid)
                )
                dependency_path = os.path.normpath(
                    os.path.join(parent_bundle_path, dep.parent_path)
                )
                if not dependency_path.startswith(parent_bundle_path) or (
                    not os.path.islink(dependency_path) and not os.path.exists(dependency_path)
                ):
                    raise Exception(
                        'Invalid dependency %s'
                        % (path_util.safe_join(dep.parent_uuid, dep.parent_path))
                    )

                child_path = os.path.normpath(os.path.join(path, dep.child_path))
                if not child_path.startswith(path):
                    raise Exception('Invalid key for dependency: %s' % (dep.child_path))

                deps.append((dependency_path, child_path))

            remove_path(path)

            if len(deps) == 1 and deps[0][1] == path:
                path_util.copy(deps[0][0], path, follow_symlinks=False)
            else:
                os.mkdir(path)
                for dependency_path, child_path in deps:
                    path_util.copy(dependency_path, child_path, follow_symlinks=False)

            self._upload_manager.update_metadata_and_save(bundle, enforce_disk_quota=True)
            logger.info('Finished making bundle %s', bundle.uuid)
            self._model.update_bundle(bundle, {'state': State.READY})
        except Exception as e:
            logger.info('Failing bundle %s: %s', bundle.uuid, str(e))
            self._model.update_bundle(
                bundle, {'state': State.FAILED, 'metadata': {'failure_message': str(e)}}
            )
        finally:
            with self._make_uuids_lock:
                self._make_uuids.remove(bundle.uuid)
Exemplo n.º 8
0
def get_target_path(bundle_store, model, target):
  '''
  Return the on-disk location of the target (bundle_spec, path) pair.
  '''
  (bundle_spec, path) = target
  uuid = get_spec_uuid(model, bundle_spec)
  bundle = model.get_bundle(uuid)
  if not bundle.data_hash:
    message = 'Unexpected: %s is ready but it has no data hash!' % (bundle,)
    precondition(bundle.state != State.READY, message)
    if bundle.state == State.FAILED:
      raise UsageError('%s failed unrecoverably' % (bundle,))
    else:
      raise UsageError('%s has not yet been executed' % (bundle,))
  bundle_root = bundle_store.get_location(bundle.data_hash)
  final_path = path_util.safe_join(bundle_root, path)
  result = path_util.TargetPath(final_path)
  result.target = target
  return result
Exemplo n.º 9
0
def get_target_path(bundle_store, model, target):
    '''
    Return the on-disk location of the target (bundle_uuid, subpath) pair.
    '''
    (uuid, path) = target
    bundle = model.get_bundle(uuid)
    if not bundle.data_hash:
        # Note that the bundle might not be done, but return the location anyway to the temporary directory
        bundle_root = get_current_location(bundle_store, uuid)
    else:
        bundle_root = bundle_store.get_location(bundle.data_hash)
    final_path = path_util.safe_join(bundle_root, path)

    # This is too restrictive because it means we can't follow any of the
    # components of a make bundle.
    #path_util.check_under_path(final_path, bundle_root)

    result = path_util.TargetPath(final_path)
    result.target = target
    return result
Exemplo n.º 10
0
    def _make_bundle(self, bundle):
        try:
            bundle_link_url = getattr(bundle.metadata, "link_url", None)
            bundle_location = bundle_link_url or self._bundle_store.get_bundle_location(
                bundle.uuid)
            path = os.path.normpath(bundle_location)

            deps = []
            parent_bundle_link_urls = self._model.get_bundle_metadata(
                [dep.parent_uuid for dep in bundle.dependencies], "link_url")
            for dep in bundle.dependencies:
                parent_bundle_link_url = parent_bundle_link_urls.get(
                    dep.parent_uuid)
                try:
                    parent_bundle_path = parent_bundle_link_url or os.path.normpath(
                        self._bundle_store.get_bundle_location(
                            dep.parent_uuid))
                except NotFoundError:
                    raise Exception('Invalid dependency %s' %
                                    (path_util.safe_join(
                                        dep.parent_uuid, dep.parent_path)))
                # TODO(Ashwin): make this logic non-fs specific.
                dependency_path = os.path.normpath(
                    os.path.join(parent_bundle_path, dep.parent_path))
                if not dependency_path.startswith(parent_bundle_path) or (
                        not os.path.islink(dependency_path)
                        and not os.path.exists(dependency_path)):
                    raise Exception('Invalid dependency %s' %
                                    (path_util.safe_join(
                                        dep.parent_uuid, dep.parent_path)))

                child_path = os.path.normpath(
                    os.path.join(path, dep.child_path))
                if not child_path.startswith(path):
                    raise Exception('Invalid key for dependency: %s' %
                                    (dep.child_path))

                deps.append((dependency_path, child_path))

            remove_path(path)

            if len(deps) == 1 and deps[0][1] == path:
                path_util.copy(deps[0][0], path, follow_symlinks=False)
            else:
                os.mkdir(path)
                for dependency_path, child_path in deps:
                    path_util.copy(dependency_path,
                                   child_path,
                                   follow_symlinks=False)

            # TODO(Ashwin): fix
            self._model.update_disk_metadata(bundle,
                                             bundle_location,
                                             enforce_disk_quota=True)
            logger.info('Finished making bundle %s', bundle.uuid)
            self._model.update_bundle(bundle, {'state': State.READY})
        except Exception as e:
            logger.info('Failing bundle %s: %s', bundle.uuid, str(e))
            self._model.update_bundle(
                bundle,
                {
                    'state': State.FAILED,
                    'metadata': {
                        'failure_message': str(e),
                        'error_traceback': traceback.format_exc(),
                    },
                },
            )
        finally:
            with self._make_uuids_lock:
                self._make_uuids.remove(bundle.uuid)