Beispiel #1
0
    def update_manifest_cache(self, force=False):
        if not self._refresh_cache and not force:
            return
        self._refresh_cache = False

        for directory in self.path:
            choices = []
            choices.extend(
                [path.join(directory, x) for x in MANIFEST_FILENAMES])
            for item in path.easy_listdir(directory):
                choices.extend([
                    path.join(directory, item, x) for x in MANIFEST_FILENAMES
                ])
                choices.extend([
                    path.join(directory, item, 'craftr', x)
                    for x in MANIFEST_FILENAMES
                ])

            for filename in map(path.norm, choices):
                if filename in self._manifest_cache:
                    continue  # don't parse a manifest that we already parsed
                if not path.isfile(filename):
                    continue
                try:
                    self.parse_manifest(filename)
                except Manifest.Invalid as exc:
                    logger.warn('invalid manifest found:', filename)
                    logger.warn(exc, indent=1)
Beispiel #2
0
    def execute(self, parser, args):
        directory = args.directory or args.name
        if path.maybedir(directory):
            directory = path.join(directory, args.name)

        if not path.exists(directory):
            logger.debug('creating directory "{}"'.format(directory))
            path.makedirs(directory)
        elif not path.isdir(directory):
            logger.error('"{}" is not a directory'.format(directory))
            return 1

        if args.nested:
            directory = path.join(directory, 'craftr')
            path.makedirs(directory)

        mfile = path.join(directory, 'manifest.' + args.format)
        sfile = path.join(directory, 'Craftrfile')
        for fn in [mfile, sfile]:
            if path.isfile(fn):
                logger.error('"{}" already exists'.format(fn))
                return 1

        logger.debug('creating file "{}"'.format(mfile))
        with open(mfile, 'w') as fp:
            if args.format == 'cson':
                lines = textwrap.dedent('''
          name: "%s"
          version: "%s"
          project_dir: ".."
          author: ""
          url: ""
          dependencies: {}
          options: {}
        ''' % (args.name, args.version)).lstrip().split('\n')
                if not args.nested:
                    del lines[2]
            elif args.format == 'json':
                lines = textwrap.dedent('''
          {
            "name": "%s",
            "version": "%s",
            "project_dir": "..",
            "author": "",
            "url": "",
            "dependencies": {},
            "options": {}
          }''' % (args.name, args.version)).lstrip().split('\n')
                if not args.nested:
                    del lines[3]
            fp.write('\n'.join(lines))

        logger.debug('creating file "{}"'.format(sfile))
        with open(sfile, 'w') as fp:
            print('# {}'.format(args.name), file=fp)
Beispiel #3
0
 def __init__(self, maindir=None):
   self.maindir = path.norm(maindir or path.getcwd())
   self.builddir = path.join(self.maindir, 'build')
   self.graph = build.Graph()
   self.path = [self.stl_dir, self.maindir, path.join(self.maindir, 'craftr/modules')]
   self.modulestack = []
   self.modules = {}
   self.options = {}
   self.cache = {'loaders': {}}
   self._tempdir = None
   self._manifest_cache = {}  # maps manifest_filename: manifest
   self._refresh_cache = True
Beispiel #4
0
    def execute(self, parser, args):
        directory = args.directory or args.name
        if path.maybedir(directory):
            directory = path.join(directory, args.name)

        if not path.exists(directory):
            logger.debug('creating directory "{}"'.format(directory))
            path.makedirs(directory)
        elif not path.isdir(directory):
            logger.error('"{}" is not a directory'.format(directory))
            return 1

        if args.nested:
            directory = path.join(directory, "craftr")
            path.makedirs(directory)

        mfile = path.join(directory, MANIFEST_FILENAME)
        sfile = path.join(directory, "Craftrfile")
        for fn in [mfile, sfile]:
            if path.isfile(fn):
                logger.error('"{}" already exists'.format(fn))
                return 1

        logger.debug('creating file "{}"'.format(mfile))
        with open(mfile, "w") as fp:
            lines = (
                textwrap.dedent(
                    """
        {
          "name": "%s",
          "version": "%s",
          "project_dir": "..",
          "author": "",
          "url": "",
          "dependencies": {},
          "options": {}
        }\n"""
                    % (args.name, args.version)
                )
                .lstrip()
                .split("\n")
            )
            if not args.nested:
                del lines[3]
            fp.write("\n".join(lines))

        logger.debug('creating file "{}"'.format(sfile))
        with open(sfile, "w") as fp:
            print("# {}".format(args.name), file=fp)
Beispiel #5
0
def load_file(filename, export_default_namespace=True):
  """
  Loads a Python file into a new module-like object and returns it. The
  *filename* is assumed relative to the currently executed module's
  directory (NOT the project directory which can be different).
  """

  module = session.module
  __name__ = module.ident + ':' + filename
  if not path.isabs(filename):
    filename = path.join(module.directory, filename)
  filename = path.norm(filename)

  with open(filename, 'r') as fp:
    code = compile(fp.read(), filename, 'exec')

  scope = Namespace()
  if export_default_namespace:
    vars(scope).update(module.get_init_globals())
    scope.__module__ = module.namespace
  scope.__file__ = filename
  scope.__name__ = __name__
  exec(code, vars(scope))

  return scope
Beispiel #6
0
  def run(self):
    """
    Loads the code of the main Craftr build script as specified in the modules
    manifest and executes it. Note that this must occur in a context where
    the :data:`session` is available.

    :raise RuntimeError: If there is no current :data:`session` or if the
      module was already executed.
    """

    if not session:
      raise RuntimeError('no current session')
    if self.executed:
      raise RuntimeError('already run')

    self.executed = True
    self.init_options()

    script_fn = path.norm(path.join(self.directory, self.manifest.main))
    with open(script_fn) as fp:
      code = compile(fp.read(), script_fn, 'exec')

    vars(self.namespace).update(self.get_init_globals())
    self.namespace.__file__ = script_fn
    self.namespace.__name__ = self.manifest.name
    self.namespace.__version__ = str(self.manifest.version)
    try:
      session.modulestack.append(self)
      exec(code, vars(self.namespace))
    finally:
      assert session.modulestack.pop() is self
Beispiel #7
0
def load_file(filename, export_default_namespace=True):
  """
  Loads a Python file into a new module-like object and returns it. The
  *filename* is assumed relative to the currently executed module's
  directory (NOT the project directory which can be different).
  """

  module = session.module
  __name__ = module.ident + ':' + filename
  if not path.isabs(filename):
    filename = path.join(module.directory, filename)
  filename = path.norm(filename)

  module.dependent_files.append(filename)
  with open(filename, 'r') as fp:
    code = compile(fp.read(), filename, 'exec')

  scope = Namespace()
  if export_default_namespace:
    vars(scope).update(module.get_init_globals())
    scope.__module__ = module.namespace
  scope.__file__ = filename
  scope.__name__ = __name__
  exec(code, vars(scope))

  return scope
Beispiel #8
0
    def export(self, writer, context, platform):
        """
    Export the target to a Ninja manifest.
    """

        writer.comment("target: {}".format(self.name))
        writer.comment("--------" + "-" * len(self.name))
        commands = platform.prepare_commands(
            [list(map(str, c)) for c in self.commands])

        # Check if we need to export a command file or can export the command
        # directly.
        if not self.environ and len(commands) == 1:
            commands = [platform.prepare_single_command(commands[0], self.cwd)]
        else:
            filename = path.join('.commands', self.name)
            command, __ = platform.write_command_file(filename,
                                                      commands,
                                                      self.inputs,
                                                      self.outputs,
                                                      cwd=self.cwd,
                                                      environ=self.environ,
                                                      foreach=self.foreach)
            commands = [command]

        assert len(commands) == 1
        command = shell.join(commands[0], for_ninja=True)

        writer.rule(self.name,
                    command,
                    pool=self.pool,
                    deps=self.deps,
                    depfile=self.depfile,
                    description=self.description)

        if self.msvc_deps_prefix:
            # We can not write msvc_deps_prefix on the rule level with Ninja
            # versions older than 1.7.1. Write it global instead, but that *could*
            # lead to issues...
            indent = 1 if context.ninja_version > '1.7.1' else 0
            writer.variable('msvc_deps_prefix', self.msvc_deps_prefix, indent)

        writer.newline()
        if self.foreach:
            assert len(self.inputs) == len(self.outputs)
            for infile, outfile in zip(self.inputs, self.outputs):
                writer.build([outfile],
                             self.name, [infile],
                             implicit=self.implicit_deps,
                             order_only=self.order_only_deps)
        else:
            writer.build(self.outputs or [self.name],
                         self.name,
                         self.inputs,
                         implicit=self.implicit_deps,
                         order_only=self.order_only_deps)

        if self.outputs and self.name not in self.outputs and not self.explicit:
            writer.build(self.name, 'phony', self.outputs)
Beispiel #9
0
 def __init__(self, maindir=None):
     self.maindir = path.norm(maindir or path.getcwd())
     self.builddir = path.join(self.maindir, 'build')
     self.graph = build.Graph()
     self.path = [
         self.stl_dir, self.stl_auxiliary_dir, self.maindir,
         path.join(self.maindir, 'craftr/modules')
     ]
     self.modulestack = []
     self.modules = {}
     self.preferred_versions = {}
     self.main_module = None
     self.options = {}
     self.cache = {}
     self.tasks = {}
     self._tempdir = None
     self._manifest_cache = {}  # maps manifest_filename: manifest
     self._refresh_cache = True
Beispiel #10
0
def buildlocal(rel_path):
  """
  Given a relative path, returns the path (still relative) to the build
  directory for the current module. This is basically a shorthand for
  prepending the module name and version to *path*.
  """

  if path.isabs(rel_path):
    return rel_path
  return path.canonical(path.join(session.module.ident, rel_path))
Beispiel #11
0
def buildlocal(rel_path):
  """
  Given a relative path, returns the path (still relative) to the build
  directory for the current module. This is basically a shorthand for
  prepending the module name and version to *path*.
  """

  if path.isabs(rel_path):
    return rel_path
  return path.canonical(path.join(session.module.ident, rel_path))
Beispiel #12
0
  def init_loader(self, recursive=False, _break_recursion=None):
    """
    Check all available loaders as defined in the :attr:`manifest` until the
    first loads successfully.

    :param recursive: Initialize the loaders of all dependencies as well.
    :raise RuntimeError: If there is no current session context.
    :raise LoaderInitializationError: If none of the loaders matched.
    """

    if not session:
      raise RuntimeError('no current session')
    if not self.manifest.loaders:
      return
    if _break_recursion is self:
      return

    if recursive:
      for name, version in self.manifest.dependencies.items():
        module = session.find_module(name, version)
        module.init_loader(True, _break_recursion=self)

    self.init_options()
    if self.loader is not None:
      return

    logger.info('running loaders for {}'.format(self.ident))
    with logger.indent():
      # Read the cached loader data and create the context.
      installdir = path.join(session.builddir, self.ident, 'src')
      cache = session.cache['loaders'].get(self.ident)
      context = LoaderContext(self.directory, self.manifest, self.options,
          installdir = installdir)
      context.get_temporary_directory = session.get_temporary_directory

      # Check all loaders in-order.
      errors = []
      for loader in self.manifest.loaders:
        logger.info('[+]', loader.name)
        with logger.indent():
          try:
            if cache and loader.name == cache['name']:
              new_data = loader.load(context, cache['data'])
            else:
              new_data = loader.load(context, None)
          except manifest.LoaderError as exc:
            errors.append(exc)
          else:
            self.loader = loader
            session.cache['loaders'][self.ident] = {
                'name': loader.name, 'data': new_data}
            break
      else:
        raise LoaderInitializationError(self, errors)
Beispiel #13
0
 def export(self, writer, context, platform):
   name = str(self)[1:]
   if not self.preamble and not self.environ:
     self.exported_command = shlex.join(self.command)
   else:
     filename = path.join('.tools', name)
     command, filename = platform.write_command_file(
         filename, list(self.preamble) + [self.command], environ=self.environ,
         accept_additional_args=True)
     self.exported_command = shell.join(command)
   writer.variable(name, self.exported_command)
Beispiel #14
0
  def export(self, writer, context, platform):
    """
    Export the target to a Ninja manifest.
    """

    writer.comment("target: {}".format(self.name))
    writer.comment("--------" + "-" * len(self.name))
    commands = platform.prepare_commands([list(map(str, c)) for c in self.commands])

    # Check if we need to export a command file or can export the command
    # directly.
    if not self.environ and len(commands) == 1:
      commands = [platform.prepare_single_command(commands[0], self.cwd)]
    else:
      filename = path.join('.commands', self.name)
      command, __ = platform.write_command_file(filename, commands,
        self.inputs, self.outputs, cwd=self.cwd, environ=self.environ,
        foreach=self.foreach)
      commands = [command]

    assert len(commands) == 1
    command = shell.join(commands[0], for_ninja=True)

    writer.rule(self.name, command, pool=self.pool, deps=self.deps,
      depfile=self.depfile, description=self.description)

    if self.msvc_deps_prefix:
      # We can not write msvc_deps_prefix on the rule level with Ninja
      # versions older than 1.7.1. Write it global instead, but that *could*
      # lead to issues...
      indent = 1 if context.ninja_version > '1.7.1' else 0
      writer.variable('msvc_deps_prefix', self.msvc_deps_prefix, indent)

    writer.newline()
    if self.foreach:
      assert len(self.inputs) == len(self.outputs)
      for infile, outfile in zip(self.inputs, self.outputs):
        writer.build(
          [outfile],
          self.name,
          [infile],
          implicit=self.implicit_deps,
          order_only=self.order_only_deps)
    else:
      writer.build(
        self.outputs or [self.name],
        self.name,
        self.inputs,
        implicit=self.implicit_deps,
        order_only=self.order_only_deps)

    if self.outputs and self.name not in self.outputs and not self.explicit:
      writer.build(self.name, 'phony', self.outputs)
Beispiel #15
0
  def update_manifest_cache(self, force=False):
    if not self._refresh_cache and not force:
      return
    self._refresh_cache = False

    for directory in self.path:
      choices = []
      choices.append(path.join(directory, MANIFEST_FILENAME))
      for item in path.easy_listdir(directory):
        choices.append(path.join(directory, item, MANIFEST_FILENAME))
        choices.append(path.join(directory, item, 'craftr', MANIFEST_FILENAME))

      for filename in map(path.norm, choices):
        if filename in self._manifest_cache:
          continue  # don't parse a manifest that we already parsed
        if not path.isfile(filename):
          continue
        try:
          self.parse_manifest(filename)
        except Manifest.Invalid as exc:
          logger.warn('invalid manifest found:', filename)
          logger.warn(exc, indent=1)
Beispiel #16
0
 def export(self, writer, context, platform):
     name = str(self)[1:]
     if not self.preamble and not self.environ:
         self.exported_command = shell.join(self.command)
     else:
         filename = path.join('.tools', name)
         command, filename = platform.write_command_file(
             filename,
             list(self.preamble) + [self.command],
             environ=self.environ,
             accept_additional_args=True)
         self.exported_command = shell.join(command)
     writer.variable(name, self.exported_command)
Beispiel #17
0
    def get_temporary_directory(self):
        """
    Returns a writable temporary directory that is primarily used by loaders
    to store temporary files. The temporary directory will be deleted when
    the Session context ends unless the ``craftr.keep_temporary_directory``
    option is set.

    :raise RuntimeError: If the session is not currently in context.
    """

        if Session.current is not self:
            raise RuntimeError('session not in context')
        if not self._tempdir:
            self._tempdir = path.join(self.builddir, '.temp')
            logger.debug('created temporary directory:', self._tempdir)
        return self._tempdir
Beispiel #18
0
def relocate_files(files, outdir, suffix, replace_suffix=True, parent=None):
  """
  Converts a list of filenames, relocating them to *outdir* and replacing
  their existing suffix. If *suffix* is a callable, it will be passed the
  new filename and expected to return the same filename, eventually with
  a different suffix.
  """

  if parent is None:
    parent = session.module.project_dir
  result = []
  for filename in files:
    filename = path.join(outdir, path.rel(filename, parent))
    filename = path.addsuffix(filename, suffix, replace=replace_suffix)
    result.append(filename)
  return result
Beispiel #19
0
  def get_temporary_directory(self):
    """
    Returns a writable temporary directory that is primarily used by loaders
    to store temporary files. The temporary directory will be deleted when
    the Session context ends unless the ``craftr.keep_temporary_directory``
    option is set.

    :raise RuntimeError: If the session is not currently in context.
    """

    if Session.current is not self:
      raise RuntimeError('session not in context')
    if not self._tempdir:
      self._tempdir = path.join(self.builddir, '.temp')
      logger.debug('created temporary directory:', self._tempdir)
    return self._tempdir
Beispiel #20
0
def relocate_files(files, outdir, suffix, replace_suffix=True, parent=None):
  """
  Converts a list of filenames, relocating them to *outdir* and replacing
  their existing suffix. If *suffix* is a callable, it will be passed the
  new filename and expected to return the same filename, eventually with
  a different suffix.
  """

  if parent is None:
    parent = session.module.namespace.project_dir
  result = []
  for filename in files:
    filename = path.join(outdir, path.rel(filename, parent))
    filename = path.addsuffix(filename, suffix, replace=replace_suffix)
    result.append(filename)
  return result
Beispiel #21
0
def load_file(filename):
  """
  Loads a Python file into a new module-like object and returns it. The
  *filename* is assumed relative to the currently executed module's
  directory (NOT the project directory which can be different).
  """

  if not path.isabs(filename):
    filename = path.join(session.module.directory, filename)

  with open(filename, 'r') as fp:
    code = compile(fp.read(), filename, 'exec')

  scope = Namespace()
  vars(scope).update(globals())
  exec(code, vars(scope))
  return scope
Beispiel #22
0
    def execute(self, parser, args):
        if hasattr(args, 'include_path'):
            session.path.extend(map(path.norm, args.include_path))

        # Help-command preprocessing. Check if we're to show the help on a builtin
        # object, otherwise extract the module name if applicable.
        if self.mode == 'help':
            if not args.name:
                help('craftr')
                return 0
            if args.name in vars(craftr.defaults):
                help(getattr(craftr.defaults, args.name))
                return 0
            # Check if we have an absolute symbol reference.
            if ':' in args.name:
                if args.module:
                    parser.error(
                        '-m/--module option conflicting with name argument: "{}"'
                        .format(args.name))
                args.module, args.name = args.name.split(':', 1)

        module = self._find_module(parser, args)
        session.main_module = module
        self.ninja_bin, self.ninja_version = get_ninja_info()

        # Create and switch to the build directory.
        session.builddir = path.abs(path.norm(args.build_dir, INIT_DIR))
        path.makedirs(session.builddir)
        os.chdir(session.builddir)
        self.cachefile = path.join(session.builddir, '.craftrcache')

        # Prepare options, loaders and execute.
        if self.mode in ('export', 'run', 'help'):
            return self._export_run_or_help(args, module)
        elif self.mode == 'dump-options':
            return self._dump_options(args, module)
        elif self.mode == 'dump-deptree':
            return self._dump_deptree(args, module)
        elif self.mode in ('build', 'clean'):
            return self._build_or_clean(args)
        elif self.mode == 'lock':
            self._create_lockfile()
        else:
            raise RuntimeError("mode: {}".format(self.mode))
Beispiel #23
0
  def run(self):
    """
    Loads the code of the main Craftr build script as specified in the modules
    manifest and executes it. Note that this must occur in a context where
    the :data:`session` is available.

    :raise RuntimeError: If there is no current :data:`session` or if the
      module was already executed.
    """

    if not session:
      raise RuntimeError('no current session')
    if self.executed:
      raise RuntimeError('already run')

    self.executed = True
    self.init_options()
    self.init_loader()

    script_fn = path.norm(path.join(self.directory, self.manifest.main))
    with open(script_fn) as fp:
      code = compile(fp.read(), script_fn, 'exec')

    from craftr import defaults
    for key, value in vars(defaults).items():
      if not key.startswith('_'):
        vars(self.namespace)[key] = value
    vars(self.namespace).update({
      '__file__': script_fn,
      '__name__': self.manifest.name,
      '__version__': str(self.manifest.version),
      'options': self.options,
      'loader': self.loader,
      'project_dir': self.project_dir,
    })

    try:
      session.modulestack.append(self)
      exec(code, vars(self.namespace))
    finally:
      assert session.modulestack.pop() is self
Beispiel #24
0
    def _download_progress(self, url, context, data):
        spinning = data["size"] is None
        if data["downloaded"] == 0:
            # If what we're trying to download already exists, we don't have
            # to redownload it.
            suffix, directory = self._get_archive_unpack_info(context, data["filename"])
            urlfile = path.join(directory, ".craftr_downloadurl")
            if path.isfile(urlfile):
                with open(urlfile) as fp:
                    if fp.read().strip() == url:
                        raise self.DownloadAlreadyExists(directory)

            logger.progress_begin("Downloading {}".format(url), spinning)
        if spinning:
            # TODO: Bytes to human readable
            logger.progress_update(None, data["downloaded"])
        else:
            progress = data["downloaded"] / data["size"]
            logger.progress_update(progress, "{}%".format(int(progress * 100)))
        if data["completed"]:
            logger.progress_end()
Beispiel #25
0
def write_response_file(arguments, builder=None, name=None, force_file=False):
  """
  Creates a response-file with the specified *name* in the in the
  ``buildfiles/`` directory and writes the *arguments* list quoted into
  the file. If *builder* is specified, it must be a :class:`TargetBuilder`
  and the response file will be added to the implicit dependencies.

  If *force_file* is set to True, a file will always be written. Otherwise,
  the function will into possible limitations of the platform and decide
  whether to write a response file or to return the *arguments* as is.

  Returns a tuple of ``(filename, arguments)``. If a response file is written,
  the returned *arguments* will be a list with a single string that is the
  filename prepended with ``@``. The *filename* part can be None if no
  response file needed to be exported.
  """

  if not name:
    if not builder:
      raise ValueError('builder must be specified if name is bot')
    name = builder.name + '.response.txt'

  if platform.name != 'win':
    return None, arguments

  # We'll just assume that there won't be more than 2048 characters for
  # other flags. The windows max buffer size is 8192.
  content = shell.join(arguments)
  if len(content) < 6144:
    return None, arguments

  filename = buildlocal(path.join('buildfiles', name))
  if builder:
    builder.implicit_deps.append(filename)

  if session.builddir:
    path.makedirs(path.dirname(filename))
    with open(filename, 'w') as fp:
      fp.write(content)
  return filename, ['@' + filename]
Beispiel #26
0
def write_response_file(arguments, builder=None, name=None, force_file=False, suffix=''):
  """
  Creates a response-file with the specified *name* in the in the
  ``buildfiles/`` directory and writes the *arguments* list quoted into
  the file. If *builder* is specified, it must be a :class:`TargetBuilder`
  and the response file will be added to the implicit dependencies.

  If *force_file* is set to True, a file will always be written. Otherwise,
  the function will into possible limitations of the platform and decide
  whether to write a response file or to return the *arguments* as is.

  Returns a tuple of ``(filename, arguments)``. If a response file is written,
  the returned *arguments* will be a list with a single string that is the
  filename prepended with ``@``. The *filename* part can be None if no
  response file needed to be exported.
  """

  if not name:
    if not builder:
      raise ValueError('builder must be specified if name is bot')
    name = builder.name + suffix + '.response.txt'

  if platform.name != 'win':
    return None, arguments

  # We'll just assume that there won't be more than 2048 characters for
  # other flags. The windows max buffer size is 8192.
  content = shell.join(arguments)
  if len(content) < 6144:
    return None, arguments

  filename = buildlocal(path.join('buildfiles', name))
  if builder:
    builder.implicit_deps.append(filename)

  if session.builddir:
    path.makedirs(path.dirname(filename))
    with open(filename, 'w') as fp:
      fp.write(content)
  return filename, ['@' + filename]
Beispiel #27
0
    def _find_module(self, parser, args):
        """
    Find the main Craftr module that is to be executed. Returns None in
    modes that do not require a main module.
    """

        if self.mode not in ('export', 'run', 'help', 'dump-options',
                             'dump-deptree'):
            return None

        # Determine the module to execute, either from the current working
        # directory or find it by name if one is specified.
        if not args.module:
            for fn in MANIFEST_FILENAMES + [
                    path.join('craftr', x) for x in MANIFEST_FILENAMES
            ]:
                if path.isfile(fn):
                    module = session.parse_manifest(fn)
                    break
            else:
                logger.error('"{}" does not exist'.format(
                    MANIFEST_FILENAMES[0]))
                sys.exit(1)
        else:
            # TODO: For some reason, prints to stdout are not visible here.
            # TODO: Prints to stderr however work fine.
            try:
                module_name, version = parse_module_spec(args.module)
            except ValueError as exc:
                logger.error(
                    '{} (note: you have to escape > and < characters)'.format(
                        exc))
                sys.exit(1)
            try:
                module = session.find_module(module_name, version)
            except Module.NotFound as exc:
                logger.error('module not found: ' + str(exc))
                sys.exit(1)

        return module
Beispiel #28
0
def download_file(url, filename=None, file=None, directory=None,
    on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None):
  """
  Download a file from a URL to one of the following destinations:

  :param filename: A filename to write the downloaded file to.
  :param file: A file-like object.
  :param directory: A directory. The filename will be automatically
    determined from the ``Content-Disposition`` header received from
    the server or the last path elemnet in the URL.

  Additional parameters for the *directory* parameter:

  :param on_exists: The operation to perform when the file already exists.
    Available modes are ``rename``, ``overwrite`` and ``skip``.

  Additional parameters:

  :param progress: A callable that accepts a single parameter that is a
    dictionary with information about the progress of the download. The
    dictionary provides the keys ``size``,  ``downloaded`` and ``response``.
    If the callable returns :const:`False` (specifically the value False), the
    download will be aborted and a :class:`UserInterrupt` will be raised.
  :param urlopen_kwargs: A dictionary with additional keyword arguments
    for :func:`urllib.request.urlopen`.

  Raise and return:

  :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
  :raise URLError: Can be raised by :func:`urllib.request.urlopen`.
  :raise UserInterrupt: If the *progress* returned :const:`False`.
  :return: If the download mode is *directory*, the name of the downloaded
    file will be returned and False if the file was newly downloaded, True
    if the download was skipped because the file already existed.
    Otherwise, the number of bytes downloaded will be returned.
  """

  argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']})

  if sum(map(bool, [filename, file, directory])) != 1:
    raise ValueError('exactly one of filename, file or directory must be specifed')

  response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))

  if directory:
    try:
      filename = parse_content_disposition(
        response.headers.get('Content-Disposition', ''))
    except ValueError:
      filename = url.split('/')[-1]
    filename = path.join(directory, filename)
    path.makedirs(directory)

    if path.exists(filename):
      if on_exists == 'skip':
        return filename, True
      elif on_exists == 'rename':
        index = 0
        while True:
          new_filename = filename + '_{:0>4}'.format(index)
          if not path.exists(new_filename):
            filename = new_filename
            break
          index += 1
      elif on_exists != 'overwrite':
        raise RuntimeError

  try:
    size = int(response.headers.get('Content-Length', ''))
  except ValueError:
    size = None

  progress_info = {'response': response, 'size': size, 'downloaded': 0,
    'completed': False, 'filename': filename, 'url': url}
  if progress and progress(progress_info) is False:
    raise UserInterrupt

  def copy_to_file(fp):
    while True:
      data = response.read(chunksize)
      if not data:
        break
      progress_info['downloaded'] += len(data)
      fp.write(data)
      if progress and progress(progress_info) is False:
        raise UserInterrupt
    progress_info['completed'] = True
    if progress and progress(progress_info) is False:
      raise UserInterrupt

  if filename:
    path.makedirs(path.dirname(filename))
    try:
      with open(filename, 'wb') as fp:
        copy_to_file(fp)
    except BaseException:
      # Delete the file if it could not be downloaded successfully.
      path.remove(filename, silent=True)
      raise
  elif file:
    copy_to_file(file)

  return filename, False
Beispiel #29
0
    def execute(self, parser, args):
        session.path.extend(map(path.norm, args.include_path))

        if self.mode == "export":
            # Determine the module to execute, either from the current working
            # directory or find it by name if one is specified.
            if not args.module:
                for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]:
                    if path.isfile(fn):
                        module = session.parse_manifest(fn)
                        break
                else:
                    parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
            else:
                # TODO: For some reason, prints to stdout are not visible here.
                # TODO: Prints to stderr however work fine.
                try:
                    module_name, version = parse_module_spec(args.module)
                except ValueError as exc:
                    parser.error("{} (note: you have to escape > and < characters)".format(exc))
                try:
                    module = session.find_module(module_name, version)
                except Module.NotFound as exc:
                    parser.error("module not found: " + str(exc))
        else:
            module = None

        ninja_bin, ninja_version = get_ninja_info()

        # Create and switch to the build directory.
        session.builddir = path.abs(args.build_dir)
        path.makedirs(session.builddir)
        os.chdir(session.builddir)

        # Read the cache and parse command-line options.
        cachefile = path.join(session.builddir, ".craftrcache")
        if not read_cache(cachefile) and self.mode != "export":
            logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode))
            logger.error("Make sure to generate a build tree with 'craftr export'")
            return 1

        # Prepare options, loaders and execute.
        if self.mode == "export":
            session.expand_relative_options(module.manifest.name)
            session.cache["build"] = {}
            try:
                module.run()
            except Module.InvalidOption as exc:
                for error in exc.format_errors():
                    logger.error(error)
                return 1
            except craftr.defaults.ModuleError as exc:
                logger.error("error:", exc)
                return 1
            finally:
                if sys.exc_info():
                    # We still want to write the cache, especially so that data already
                    # loaded with loaders doesn't need to be re-loaded. They'll find out
                    # when the cached information was not valid.
                    write_cache(cachefile)

            # Write the cache back.
            session.cache["build"]["targets"] = list(session.graph.targets.keys())
            session.cache["build"]["main"] = module.ident
            session.cache["build"]["options"] = args.options
            write_cache(cachefile)

            # Write the Ninja manifest.
            with open("build.ninja", "w") as fp:
                platform = core.build.get_platform_helper()
                context = core.build.ExportContext(ninja_version)
                writer = core.build.NinjaWriter(fp)
                session.graph.export(writer, context, platform)

        else:
            parse_cmdline_options(session.cache["build"]["options"])
            main = session.cache["build"]["main"]
            available_targets = frozenset(session.cache["build"]["targets"])

            logger.debug("build main module:", main)
            session.expand_relative_options(get_volatile_module_version(main)[0])

            # Check the targets and if they exist.
            targets = []
            for target in args.targets:
                if "." not in target:
                    target = main + "." + target
                elif target.startswith("."):
                    target = main + target

                module_name, target = target.rpartition(".")[::2]
                module_name, version = get_volatile_module_version(module_name)
                ref_module = session.find_module(module_name, version or "*")
                target = craftr.targetbuilder.get_full_name(target, ref_module)
                if target not in available_targets:
                    parser.error("no such target: {}".format(target))
                targets.append(target)

            # Execute the ninja build.
            cmd = [ninja_bin]
            if args.verbose:
                cmd += ["-v"]
            if self.mode == "clean":
                cmd += ["-t", "clean"]
                if not args.recursive:
                    cmd += ["-r"]
            cmd += targets
            return shell.run(cmd).returncode
Beispiel #30
0
def main():
    # Create argument parsers and dynamically include all BaseCommand
    # subclasses into it.
    parser = argparse.ArgumentParser(prog="craftr", description="The Craftr build system")
    parser.add_argument("-v", "--verbose", action="store_true")
    parser.add_argument("-q", "--quiet", action="store_true")
    parser.add_argument("-c", "--config", action="append", default=[])
    parser.add_argument("-C", "--no-config", action="store_true")
    parser.add_argument("-d", "--option", dest="options", action="append", default=[])
    subparsers = parser.add_subparsers(dest="command")

    commands = {
        "clean": BuildCommand("clean"),
        "build": BuildCommand("build"),
        "export": BuildCommand("export"),
        "startpackage": StartpackageCommand(),
        "version": VersionCommand(),
    }

    for key, cmd in commands.items():
        cmd.build_parser(subparsers.add_parser(key))

    # Parse the arguments.
    args = parser.parse_args()
    if not args.command:
        parser.print_usage()
        return 0

    if args.verbose:
        logger.set_level(logger.DEBUG)
    elif args.quiet:
        logger.set_level(logger.WARNING)

    session = Session()

    # Parse the user configuration file.
    try:
        config_filename = path.expanduser("~/" + CONFIG_FILENAME)
        session.options = read_config_file(config_filename)
    except FileNotFoundError as exc:
        session.options = {}
    except InvalidConfigError as exc:
        parser.error(exc)
        return 1

    # Parse the local configuration file or the ones specified on command-line.
    if not args.no_config:
        try:
            for filename in args.config:
                session.options.update(read_config_file(filename))
            if not args.config:
                choices = [CONFIG_FILENAME, path.join("craftr", CONFIG_FILENAME)]
                for fn in choices:
                    try:
                        session.options.update(read_config_file(fn))
                    except FileNotFoundError as exc:
                        pass
        except InvalidConfigError as exc:
            parser.error(exc)
            return 1

    # Execute the command in the session context.
    with session:
        parse_cmdline_options(args.options)
        return commands[args.command].execute(parser, args)
Beispiel #31
0
def download_file(url,
                  filename=None,
                  file=None,
                  directory=None,
                  on_exists='rename',
                  progress=None,
                  chunksize=4096,
                  urlopen_kwargs=None):
    """
  Download a file from a URL to one of the following destinations:

  :param filename: A filename to write the downloaded file to.
  :param file: A file-like object.
  :param directory: A directory. The filename will be automatically
    determined from the ``Content-Disposition`` header received from
    the server or the last path elemnet in the URL.

  Additional parameters for the *directory* parameter:

  :param on_exists: The operation to perform when the file already exists.
    Available modes are ``rename``, ``overwrite`` and ``skip``.

  Additional parameters:

  :param progress: A callable that accepts a single parameter that is a
    dictionary with information about the progress of the download. The
    dictionary provides the keys ``size``,  ``downloaded`` and ``response``.
    If the callable returns :const:`False` (specifically the value False), the
    download will be aborted and a :class:`UserInterrupt` will be raised.
  :param urlopen_kwargs: A dictionary with additional keyword arguments
    for :func:`urllib.request.urlopen`.

  Raise and return:

  :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
  :raise URLError: Can be raised by :func:`urllib.request.urlopen`.
  :raise UserInterrupt: If the *progress* returned :const:`False`.
  :return: If the download mode is *directory*, the name of the downloaded
    file will be returned and False if the file was newly downloaded, True
    if the download was skipped because the file already existed.
    Otherwise, the number of bytes downloaded will be returned.
  """

    argspec.validate('on_exists', on_exists,
                     {'enum': ['rename', 'overwrite', 'skip']})

    if sum(map(bool, [filename, file, directory])) != 1:
        raise ValueError(
            'exactly one of filename, file or directory must be specifed')

    response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))

    if directory:
        try:
            filename = parse_content_disposition(
                response.headers.get('Content-Disposition', ''))
        except ValueError:
            filename = url.split('/')[-1]
        filename = path.join(directory, filename)
        path.makedirs(directory)

        if path.exists(filename):
            if on_exists == 'skip':
                return filename, True
            elif on_exists == 'rename':
                index = 0
                while True:
                    new_filename = filename + '_{:0>4}'.format(index)
                    if not path.exists(new_filename):
                        filename = new_filename
                        break
                    index += 1
            elif on_exists != 'overwrite':
                raise RuntimeError

    try:
        size = int(response.headers.get('Content-Length', ''))
    except ValueError:
        size = None

    progress_info = {
        'response': response,
        'size': size,
        'downloaded': 0,
        'completed': False,
        'filename': filename,
        'url': url
    }
    if progress and progress(progress_info) is False:
        raise UserInterrupt

    def copy_to_file(fp):
        while True:
            data = response.read(chunksize)
            if not data:
                break
            progress_info['downloaded'] += len(data)
            fp.write(data)
            if progress and progress(progress_info) is False:
                raise UserInterrupt
        progress_info['completed'] = True
        if progress and progress(progress_info) is False:
            raise UserInterrupt

    if filename:
        path.makedirs(path.dirname(filename))
        try:
            with open(filename, 'wb') as fp:
                copy_to_file(fp)
        except BaseException:
            # Delete the file if it could not be downloaded successfully.
            path.remove(filename, silent=True)
            raise
    elif file:
        copy_to_file(file)

    return filename, False
Beispiel #32
0
def main():
  # Create argument parsers and dynamically include all BaseCommand
  # subclasses into it.
  parser = argparse.ArgumentParser(prog='craftr', description='The Craftr build system')
  parser.add_argument('-v', '--verbose', action='store_true')
  parser.add_argument('-q', '--quiet', action='store_true')
  parser.add_argument('-c', '--config', action='append', default=[])
  parser.add_argument('-C', '--no-config', action='store_true')
  parser.add_argument('-d', '--option', dest='options', action='append', default=[])
  subparsers = parser.add_subparsers(dest='command')

  commands = {
    'export': ExportOrBuildCommand(is_export=True),
    'build': ExportOrBuildCommand(is_export=False),
    'startpackage': StartpackageCommand()
  }
  for key, cmd in commands.items():
    cmd.build_parser(subparsers.add_parser(key))

  # Parse the arguments.
  args = parser.parse_args()
  if not args.command:
    parser.print_usage()
    return 0

  if args.verbose:
    logger.set_level(logger.DEBUG)
  elif args.quiet:
    logger.set_level(logger.WARNING)

  session = Session()

  # Parse the user configuration file.
  try:
    config_filename = path.expanduser('~/' + CONFIG_FILENAME)
    session.options = read_config_file(config_filename)
  except FileNotFoundError as exc:
    session.options = {}
  except InvalidConfigError as exc:
    parser.error(exc)
    return 1

  # Parse the local configuration file or the ones specified on command-line.
  if not args.no_config:
    try:
      for filename in args.config:
        session.options.update(read_config_file(filename))
      if not args.config:
        choices = [CONFIG_FILENAME, path.join('craftr', CONFIG_FILENAME)]
        for fn in choices:
          try:
            session.options.update(read_config_file(fn))
          except FileNotFoundError as exc:
            pass
    except InvalidConfigError as exc:
      parser.error(exc)
      return 1

  # Execute the command in the session context.
  with session:
    parse_cmdline_options(args.options)
    return commands[args.command].execute(parser, args)
Beispiel #33
0
 def project_dir(self):
     return path.norm(path.join(self.directory, self.manifest.project_dir))
Beispiel #34
0
 def scriptfile(self):
     return path.norm(path.join(self.directory, self.manifest.main))
Beispiel #35
0
  def execute(self, parser, args):
    session.path.extend(map(path.norm, args.include_path))

    if self.is_export:
      # Determine the module to execute, either from the current working
      # directory or find it by name if one is specified.
      if not args.module:
        for fn in [MANIFEST_FILENAME, path.join('craftr', MANIFEST_FILENAME)]:
          if path.isfile(fn):
            module = session.parse_manifest(fn)
            break
        else:
          parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
      else:
        # TODO: For some reason, prints to stdout are not visible here.
        # TODO: Prints to stderr however work fine.
        try:
          module_name, version = parse_module_spec(args.module)
        except ValueError as exc:
          parser.error('{} (note: you have to escape > and < characters)'.format(exc))
        try:
          module = session.find_module(module_name, version)
        except Module.NotFound as exc:
          parser.error('module not found: ' + str(exc))
    else:
      module = None

    ninja_bin, ninja_version = get_ninja_info()

    # Create and switch to the build directory.
    session.builddir = path.abs(args.build_dir)
    path.makedirs(session.builddir)
    os.chdir(session.builddir)

    # Read the cache and parse command-line options.
    cachefile = path.join(session.builddir, '.craftrcache')
    if not read_cache(cachefile) and not self.is_export:
      logger.error('Unable to load "{}", can not build'.format(cachefile))
      return 1

    # Prepare options, loaders and execute.
    if self.is_export:
      session.cache['build'] = {}
      try:
        write_cache(cachefile)
        module.run()
      except (Module.InvalidOption, Module.LoaderInitializationError) as exc:
        for error in exc.format_errors():
          logger.error(error)
        return 1
      except craftr.defaults.ModuleError as exc:
        logger.error(exc)
        return 1

      # Write the cache back.
      session.cache['build']['targets'] = list(session.graph.targets.keys())
      session.cache['build']['main'] = module.ident
      session.cache['build']['options'] = args.options
      write_cache(cachefile)

      # Write the Ninja manifest.
      with open("build.ninja", 'w') as fp:
        platform = core.build.get_platform_helper()
        context = core.build.ExportContext(ninja_version)
        writer = core.build.NinjaWriter(fp)
        session.graph.export(writer, context, platform)

    else:
      parse_cmdline_options(session.cache['build']['options'])
      main = session.cache['build']['main']
      available_targets = frozenset(session.cache['build']['targets'])

      # Check the targets and if they exist.
      targets = []
      for target in args.targets:
        if '.' not in target:
          target = main + '.' + target
        elif target.startswith('.'):
          target = main + target

        module_name, target = target.rpartition('.')[::2]
        module_name, version = get_volatile_module_version(module_name)
        ref_module = session.find_module(module_name, version or '*')
        target = craftr.targetbuilder.get_full_name(target, ref_module)
        if target not in available_targets:
          parser.error('no such target: {}'.format(target))
        targets.append(target)

      # Execute the ninja build.
      cmd = [ninja_bin]
      if args.verbose:
        cmd += ['-v']
      cmd += targets
      shell.run(cmd)
Beispiel #36
0
 def __call__(self, value):
     from craftr.core.session import session
     if not path.isabs(value):
         value = path.join(session.maindir, value)
     return path.norm(value)
Beispiel #37
0
 def __call__(self, value):
   from craftr.core.session import session
   if not path.isabs(value):
     value = path.join(session.maindir, value)
   return path.norm(value)
Beispiel #38
0
 def project_dir(self):
   return path.norm(path.join(self.directory, self.manifest.project_dir))
Beispiel #39
0
 def _get_archive_unpack_info(self, context, archive):
     suffix = nr.misc.archive.get_opener(archive)[0]
     filename = path.basename(archive)[: -len(suffix)]
     directory = path.join(context.installdir, filename)
     return suffix, directory
Beispiel #40
0
    def _export_run_or_help(self, args, module):
        """
    Called when the mode is 'export' or 'run'. Will execute the specified
    *module* and eventually export a Ninja manifest and Cache.
    """

        read_cache(False)

        session.expand_relative_options()
        session.cache['build'] = {}

        # Load the dependency lock information if it exists.
        deplock_fn = path.join(path.dirname(module.manifest.filename),
                               '.dependency-lock')
        if os.path.isfile(deplock_fn):
            with open(deplock_fn) as fp:
                session.preferred_versions = cson.load(fp)
                logger.debug('note: dependency lock file "{}" loaded'.format(
                    deplock_fn))

        try:
            module.run()
        except Module.InvalidOption as exc:
            for error in exc.format_errors():
                logger.error(error)
            return 1
        except craftr.defaults.ModuleError as exc:
            logger.error('error:', exc)
            return 1
        finally:
            if sys.exc_info() and self.mode == 'export':
                # We still want to write the cache, especially so that data already
                # loaded with loaders doesn't need to be re-loaded. They'll find out
                # when the cached information was not valid.
                write_cache(self.cachefile)

        # Fill the cache.
        session.cache['build']['targets'] = list(session.graph.targets.keys())
        session.cache['build']['modules'] = serialise_loaded_module_info()
        session.cache['build']['main'] = module.ident
        session.cache['build']['options'] = args.options
        session.cache['build']['dependency_lock_filename'] = deplock_fn

        if self.mode == 'export':
            # Add the Craftr_run_command variable which is necessary for tasks
            # to properly executed.
            run_command = ['craftr', '-q', '-P', path.rel(session.maindir)]
            if args.no_config: run_command += ['-C']
            run_command += ['-c' + x for x in args.config]
            run_command += ['run']
            if args.module: run_command += ['-m', args.module]
            run_command += ['-i' + x for x in args.include_path]
            run_command += ['-b', path.rel(session.builddir)]
            session.graph.vars['Craftr_run_command'] = shell.join(run_command)

            write_cache(self.cachefile)

            # Write the Ninja manifest.
            with open("build.ninja", 'w') as fp:
                platform = core.build.get_platform_helper()
                context = core.build.ExportContext(self.ninja_version)
                writer = core.build.NinjaWriter(fp)
                session.graph.export(writer, context, platform)
                logger.info('exported "build.ninja"')

            return 0

        elif self.mode == 'run':
            if args.task:
                if args.task not in session.graph.tasks:
                    logger.error('no such task exists: "{}"'.format(args.task))
                    return 1
                task = session.graph.tasks[args.task]
                return task.invoke(args.task_args)
            return 0

        elif self.mode == 'help':
            if args.name not in vars(module.namespace):
                logger.error('symbol not found: "{}:{}"'.format(
                    module.manifest.name, args.name))
                return 1
            help(getattr(module.namespace, args.name))
            return 0

        assert False, "unhandled mode: {}".format(self.mode)
Beispiel #41
0
    def load(self, context, cache):
        if cache is not None and path.isdir(cache.get("directory", "")):
            # Check if the requested version changes.
            url_template = context.expand_variables(cache.get("url_template", ""))
            if url_template == cache.get("url"):
                self.directory = cache["directory"]
                logger.info("Reusing cached directory: {}".format(path.rel(self.directory, nopar=True)))
                return cache
            else:
                logger.info("Cached URL is outdated:", cache.get("url"))

        directory = None
        archive = None
        delete_after_extract = True
        for url_template in self.urls:
            url = context.expand_variables(url_template)
            if not url:
                continue
            if url.startswith("file://"):
                name = url[7:]
                if path.isdir(name):
                    logger.info("Using directory", url)
                    directory = name
                    break
                elif path.isfile(name):
                    logger.info("Using archive", url)
                    archive = name
                    delete_after_extract = False
                    break
                error = None
            else:
                error = None
                try:
                    progress = lambda d: self._download_progress(url, context, d)
                    archive, reused = httputils.download_file(
                        url, directory=context.get_temporary_directory(), on_exists="skip", progress=progress
                    )
                except (httputils.URLError, httputils.HTTPError) as exc:
                    error = exc
                except self.DownloadAlreadyExists as exc:
                    directory = exc.directory
                    logger.info("Reusing existing directory", directory)
                else:
                    if reused:
                        logger.info("Reusing cached download", path.basename(archive))
                    break

            if error:
                logger.info("Error reading", url, ":", error)

        if directory or archive:
            logger.debug("URL applies: {}".format(url))

        if not directory and archive:
            suffix, directory = self._get_archive_unpack_info(context, archive)
            logger.info(
                'Unpacking "{}" to "{}" ...'.format(path.rel(archive, nopar=True), path.rel(directory, nopar=True))
            )
            nr.misc.archive.extract(
                archive,
                directory,
                suffix=suffix,
                unpack_single_dir=True,
                check_extract_file=self._check_extract_file,
                progress_callback=self._extract_progress,
            )
        elif not directory:
            raise LoaderError(self, "no URL matched")

        self.directory = directory
        with open(path.join(self.directory, ".craftr_downloadurl"), "w") as fp:
            fp.write(url)
        return {"directory": directory, "url_template": url_template, "url": url}
Beispiel #42
0
def main():
    # Create argument parsers and dynamically include all BaseCommand
    # subclasses into it.
    parser = argparse.ArgumentParser(prog='craftr',
                                     description='The Craftr build system')
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('-q', '--quiet', action='store_true')
    parser.add_argument('-c', '--config', action='append', default=[])
    parser.add_argument('-C', '--no-config', action='store_true')
    parser.add_argument('-P', '--project-dir')
    parser.add_argument('-d',
                        '--option',
                        dest='options',
                        action='append',
                        default=[])
    subparsers = parser.add_subparsers(dest='command')

    commands = {
        'lock': BuildCommand('lock'),
        'clean': BuildCommand('clean'),
        'build': BuildCommand('build'),
        'export': BuildCommand('export'),
        'run': BuildCommand('run'),
        'help': BuildCommand('help'),
        'options': BuildCommand('dump-options'),
        'deptree': BuildCommand('dump-deptree'),
        'startpackage': StartpackageCommand(),
        'version': VersionCommand()
    }

    for key, cmd in commands.items():
        cmd.build_parser(subparsers.add_parser(key))

    # Parse the arguments.
    args = parser.parse_args()
    if not args.command:
        parser.print_usage()
        return 0

    if args.project_dir:
        os.chdir(args.project_dir)
    if args.verbose:
        logger.set_level(logger.DEBUG)
    elif args.quiet:
        logger.set_level(logger.WARNING)

    session = Session()

    # Parse the user configuration file.
    try:
        config_filename = path.expanduser('~/' + CONFIG_FILENAME)
        session.options = read_config_file(config_filename)
    except FileNotFoundError as exc:
        session.options = {}
    except InvalidConfigError as exc:
        parser.error(exc)
        return 1

    # Parse the local configuration file or the ones specified on command-line.
    if not args.no_config:
        try:
            for filename in args.config:
                session.options.update(read_config_file(filename))
            if not args.config:
                choices = [
                    CONFIG_FILENAME,
                    path.join('craftr', CONFIG_FILENAME)
                ]
                for fn in choices:
                    try:
                        session.options.update(read_config_file(fn))
                    except FileNotFoundError as exc:
                        pass
        except InvalidConfigError as exc:
            parser.error(exc)
            return 1

    # Execute the command in the session context.
    with session:
        parse_cmdline_options(args.options)
        return commands[args.command].execute(parser, args)
Beispiel #43
0
class Session(object):
    """
  This class manages the :class:`build.Graph` and loading of Craftr modules.

  .. attribute:: graph

    A :class:`build.Graph` instance.

  .. attribute:: path

    A list of paths that will be searched for Craftr modules.

  .. attribute:: module

    The Craftr module that is currently being executed. This is an instance
    of the :class:`Module` class and the same as the tip of the
    :attr:`modulestack`.

  .. attribute:: modulestack

    A list of modules where the last element (tip) is the module that is
    currently being executed.

  .. attribute:: modules

    A nested dictionary that maps from name to a dictionary of version
    numbers mapping to :class:`Module` objects. These are the modules that
    have already been loaded into the session or that have been found and
    cached but not yet been executed.

  .. attribute:: preferred_versions

    A nested dictionary with the same structure as :attr:`modules`. This
    dictionary might have been loaded from a dependency lock file and specifies
    the preferred version to load for a specific module, assuming that the
    criteria specified in the loading module's manifest is less strict. Note
    that Craftr will error if a preferred version can not be found.

  .. attribute:: maindir

    The main directory from which Craftr was run. Craftr will switch to the
    build directory at a later point, which is why we keep this member for
    reference.

  .. attribute:: builddir

    The absolute path to the build directory.

  .. attribute:: main_module

    The main :class:`Module`.

  .. attribute:: options

    A dictionary of options that are passed down to Craftr modules.

  .. attributes:: cache

    A JSON object that will be loaded from the current workspace's cache
    file and written back when Craftr exits without errors. The cache can
    contain anything and can be modified by everything, however it should
    be assured that no name conflicts and accidental modifications/deletes
    occur.

    Reserved keywords in the cache are ``"build"`` and ``"loaders"``.
  """

    #: The current session object. Create it with :meth:`start` and destroy
    #: it with :meth:`end`.
    current = None

    #: Diretory that contains the Craftr standard library.
    stl_dir = path.norm(path.join(__file__, '../../stl'))
    stl_auxiliary_dir = path.norm(path.join(__file__, '../../stl_auxiliary'))

    def __init__(self, maindir=None):
        self.maindir = path.norm(maindir or path.getcwd())
        self.builddir = path.join(self.maindir, 'build')
        self.graph = build.Graph()
        self.path = [
            self.stl_dir, self.stl_auxiliary_dir, self.maindir,
            path.join(self.maindir, 'craftr/modules')
        ]
        self.modulestack = []
        self.modules = {}
        self.preferred_versions = {}
        self.main_module = None
        self.options = {}
        self.cache = {}
        self.tasks = {}
        self._tempdir = None
        self._manifest_cache = {}  # maps manifest_filename: manifest
        self._refresh_cache = True

    def __enter__(self):
        if Session.current:
            raise RuntimeError('a session was already created')
        Session.current = self
        return Session.current

    def __exit__(self, exc_value, exc_type, exc_tb):
        if Session.current is not self:
            raise RuntimeError('session not in context')
        if self._tempdir and not self.options.get(
                'craftr.keep_temporary_directory'):
            logger.debug('removing temporary directory:', self._tempdir)
            try:
                path.remove(self._tempdir, recursive=True)
            except OSError as exc:
                logger.debug('error:', exc, indent=1)
            finally:
                self._tempdir = None
        Session.current = None

    @property
    def module(self):
        if self.modulestack:
            return self.modulestack[-1]
        return None

    def read_cache(self, fp):
        cache = json.load(fp)
        if not isinstance(cache, dict):
            raise ValueError(
                'Craftr Session cache must be a JSON object, got {}'.format(
                    type(cache).__name__))
        self.cache = cache

    def write_cache(self, fp):
        json.dump(self.cache, fp, indent='\t')

    def expand_relative_options(self, module_name=None):
        """
    After the main module has been detected, relative option names (starting
    with ``.``) should be converted to absolute option names. This is what
    the method does.
    """

        if not module_name and not self.main_module:
            raise RuntimeError('main_module not set')
        if not module_name:
            module_name = self.main_module.manifest.name

        for key in tuple(self.options.keys()):
            if key.startswith('.'):
                self.options[module_name + key] = self.options.pop(key)

    def get_temporary_directory(self):
        """
    Returns a writable temporary directory that is primarily used by loaders
    to store temporary files. The temporary directory will be deleted when
    the Session context ends unless the ``craftr.keep_temporary_directory``
    option is set.

    :raise RuntimeError: If the session is not currently in context.
    """

        if Session.current is not self:
            raise RuntimeError('session not in context')
        if not self._tempdir:
            self._tempdir = path.join(self.builddir, '.temp')
            logger.debug('created temporary directory:', self._tempdir)
        return self._tempdir

    def parse_manifest(self, filename):
        """
    Parse a manifest by filename and add register the module to the module
    cache. Returns the :class:`Module` object. If the manifest has already
    been parsed, it will not be re-parsed.

    :raise Manifest.Invalid: If the manifest is invalid.
    :return: :const:`None` if the manifest is a duplicate of an already
      parsed manifest (determined by name and version), otherwise the
      :class:`Module` object for the manifest's module.
    """

        filename = path.norm(path.abs(filename))
        if filename in self._manifest_cache:
            manifest = self._manifest_cache[filename]
            return self.find_module(manifest.name, manifest.version)

        manifest = Manifest.parse(filename)
        self._manifest_cache[filename] = manifest
        versions = self.modules.setdefault(manifest.name, {})
        if manifest.version in versions:
            other = versions[manifest.version].manifest.filename
            logger.debug('multiple occurences of "{}-{}" found\n'
                         '  - {}\n  - {}'.format(manifest.name,
                                                 manifest.version, filename,
                                                 other))
            module = None
        else:
            logger.debug('parsed manifest: {}-{} ({})'.format(
                manifest.name, manifest.version, filename))
            module = Module(path.dirname(filename), manifest)
            versions[manifest.version] = module

        return module

    def update_manifest_cache(self, force=False):
        if not self._refresh_cache and not force:
            return
        self._refresh_cache = False

        for directory in self.path:
            choices = []
            choices.extend(
                [path.join(directory, x) for x in MANIFEST_FILENAMES])
            for item in path.easy_listdir(directory):
                choices.extend([
                    path.join(directory, item, x) for x in MANIFEST_FILENAMES
                ])
                choices.extend([
                    path.join(directory, item, 'craftr', x)
                    for x in MANIFEST_FILENAMES
                ])

            for filename in map(path.norm, choices):
                if filename in self._manifest_cache:
                    continue  # don't parse a manifest that we already parsed
                if not path.isfile(filename):
                    continue
                try:
                    self.parse_manifest(filename)
                except Manifest.Invalid as exc:
                    logger.warn('invalid manifest found:', filename)
                    logger.warn(exc, indent=1)

    def find_module(self, name, version, resolve_preferred_version=True):
        """
    Finds a module in the :attr:`path` matching the specified *name* and
    *version*.

    :param name: The name of the module.
    :param version: A :class:`VersionCriteria`, :class:`Version` or string
      in a VersionCritiera format.
    :param resolve_preferred_version: If this parameter is True (default)
      and a preferred version is specified in :attr:`preferred_versions`,
      that preferred version is loaded or :class:`ModuleNotFound` is raised.
    :raise ModuleNotFound: If the module can not be found.
    :return: :class:`Module`
    """

        argspec.validate('name', name, {'type': str})
        argspec.validate('version', version,
                         {'type': [str, Version, VersionCriteria]})

        if name in renames.renames:
            logger.warn('"{}" is deprecated, use "{}" instead'.format(
                name, renames.renames[name]))
            name = renames.renames[name]

        if isinstance(version, str):
            try:
                version = Version(version)
            except ValueError as exc:
                version = VersionCriteria(version)

        if session.module and resolve_preferred_version:
            data = self.preferred_versions.get(session.module.manifest.name)
            if data is not None:
                versions = data.get(str(session.module.manifest.version))
                if versions is not None:
                    preferred_version = versions.get(name)
                    if preferred_version is not None:
                        version = Version(preferred_version)
                        logger.debug(
                            'note: loading preferred version {} of module "{}" '
                            'requested by module "{}"'.format(
                                version, name, session.module.ident))

        self.update_manifest_cache()
        if name in self.modules:
            if isinstance(version, Version):
                if version in self.modules[name]:
                    return self.modules[name][version]
                raise ModuleNotFound(name, version)
            for module in sorted(self.modules[name].values(),
                                 key=lambda x: x.manifest.version,
                                 reverse=True):
                if version(module.manifest.version):
                    return module

        raise ModuleNotFound(name, version)