Пример #1
0
    def update_manifest_cache(self, force=False):
        if not self._refresh_cache and not force:
            return
        self._refresh_cache = False

        for directory in self.path:
            choices = []
            choices.extend(
                [path.join(directory, x) for x in MANIFEST_FILENAMES])
            for item in path.easy_listdir(directory):
                choices.extend([
                    path.join(directory, item, x) for x in MANIFEST_FILENAMES
                ])
                choices.extend([
                    path.join(directory, item, 'craftr', x)
                    for x in MANIFEST_FILENAMES
                ])

            for filename in map(path.norm, choices):
                if filename in self._manifest_cache:
                    continue  # don't parse a manifest that we already parsed
                if not path.isfile(filename):
                    continue
                try:
                    self.parse_manifest(filename)
                except Manifest.Invalid as exc:
                    logger.warn('invalid manifest found:', filename)
                    logger.warn(exc, indent=1)
Пример #2
0
    def execute(self, parser, args):
        directory = args.directory or args.name
        if path.maybedir(directory):
            directory = path.join(directory, args.name)

        if not path.exists(directory):
            logger.debug('creating directory "{}"'.format(directory))
            path.makedirs(directory)
        elif not path.isdir(directory):
            logger.error('"{}" is not a directory'.format(directory))
            return 1

        if args.nested:
            directory = path.join(directory, 'craftr')
            path.makedirs(directory)

        mfile = path.join(directory, 'manifest.' + args.format)
        sfile = path.join(directory, 'Craftrfile')
        for fn in [mfile, sfile]:
            if path.isfile(fn):
                logger.error('"{}" already exists'.format(fn))
                return 1

        logger.debug('creating file "{}"'.format(mfile))
        with open(mfile, 'w') as fp:
            if args.format == 'cson':
                lines = textwrap.dedent('''
          name: "%s"
          version: "%s"
          project_dir: ".."
          author: ""
          url: ""
          dependencies: {}
          options: {}
        ''' % (args.name, args.version)).lstrip().split('\n')
                if not args.nested:
                    del lines[2]
            elif args.format == 'json':
                lines = textwrap.dedent('''
          {
            "name": "%s",
            "version": "%s",
            "project_dir": "..",
            "author": "",
            "url": "",
            "dependencies": {},
            "options": {}
          }''' % (args.name, args.version)).lstrip().split('\n')
                if not args.nested:
                    del lines[3]
            fp.write('\n'.join(lines))

        logger.debug('creating file "{}"'.format(sfile))
        with open(sfile, 'w') as fp:
            print('# {}'.format(args.name), file=fp)
Пример #3
0
    def execute(self, parser, args):
        directory = args.directory or args.name
        if path.maybedir(directory):
            directory = path.join(directory, args.name)

        if not path.exists(directory):
            logger.debug('creating directory "{}"'.format(directory))
            path.makedirs(directory)
        elif not path.isdir(directory):
            logger.error('"{}" is not a directory'.format(directory))
            return 1

        if args.nested:
            directory = path.join(directory, "craftr")
            path.makedirs(directory)

        mfile = path.join(directory, MANIFEST_FILENAME)
        sfile = path.join(directory, "Craftrfile")
        for fn in [mfile, sfile]:
            if path.isfile(fn):
                logger.error('"{}" already exists'.format(fn))
                return 1

        logger.debug('creating file "{}"'.format(mfile))
        with open(mfile, "w") as fp:
            lines = (
                textwrap.dedent(
                    """
        {
          "name": "%s",
          "version": "%s",
          "project_dir": "..",
          "author": "",
          "url": "",
          "dependencies": {},
          "options": {}
        }\n"""
                    % (args.name, args.version)
                )
                .lstrip()
                .split("\n")
            )
            if not args.nested:
                del lines[3]
            fp.write("\n".join(lines))

        logger.debug('creating file "{}"'.format(sfile))
        with open(sfile, "w") as fp:
            print("# {}".format(args.name), file=fp)
Пример #4
0
    def _download_progress(self, url, context, data):
        spinning = data["size"] is None
        if data["downloaded"] == 0:
            # If what we're trying to download already exists, we don't have
            # to redownload it.
            suffix, directory = self._get_archive_unpack_info(context, data["filename"])
            urlfile = path.join(directory, ".craftr_downloadurl")
            if path.isfile(urlfile):
                with open(urlfile) as fp:
                    if fp.read().strip() == url:
                        raise self.DownloadAlreadyExists(directory)

            logger.progress_begin("Downloading {}".format(url), spinning)
        if spinning:
            # TODO: Bytes to human readable
            logger.progress_update(None, data["downloaded"])
        else:
            progress = data["downloaded"] / data["size"]
            logger.progress_update(progress, "{}%".format(int(progress * 100)))
        if data["completed"]:
            logger.progress_end()
Пример #5
0
    def _find_module(self, parser, args):
        """
    Find the main Craftr module that is to be executed. Returns None in
    modes that do not require a main module.
    """

        if self.mode not in ('export', 'run', 'help', 'dump-options',
                             'dump-deptree'):
            return None

        # Determine the module to execute, either from the current working
        # directory or find it by name if one is specified.
        if not args.module:
            for fn in MANIFEST_FILENAMES + [
                    path.join('craftr', x) for x in MANIFEST_FILENAMES
            ]:
                if path.isfile(fn):
                    module = session.parse_manifest(fn)
                    break
            else:
                logger.error('"{}" does not exist'.format(
                    MANIFEST_FILENAMES[0]))
                sys.exit(1)
        else:
            # TODO: For some reason, prints to stdout are not visible here.
            # TODO: Prints to stderr however work fine.
            try:
                module_name, version = parse_module_spec(args.module)
            except ValueError as exc:
                logger.error(
                    '{} (note: you have to escape > and < characters)'.format(
                        exc))
                sys.exit(1)
            try:
                module = session.find_module(module_name, version)
            except Module.NotFound as exc:
                logger.error('module not found: ' + str(exc))
                sys.exit(1)

        return module
Пример #6
0
  def update_manifest_cache(self, force=False):
    if not self._refresh_cache and not force:
      return
    self._refresh_cache = False

    for directory in self.path:
      choices = []
      choices.append(path.join(directory, MANIFEST_FILENAME))
      for item in path.easy_listdir(directory):
        choices.append(path.join(directory, item, MANIFEST_FILENAME))
        choices.append(path.join(directory, item, 'craftr', MANIFEST_FILENAME))

      for filename in map(path.norm, choices):
        if filename in self._manifest_cache:
          continue  # don't parse a manifest that we already parsed
        if not path.isfile(filename):
          continue
        try:
          self.parse_manifest(filename)
        except Manifest.Invalid as exc:
          logger.warn('invalid manifest found:', filename)
          logger.warn(exc, indent=1)
Пример #7
0
def read_config_file(filename, basedir=None, follow_include_directives=True):
    """
  Reads a configuration file and returns a dictionary of the values that
  it contains. The format is standard :mod:`configparser` ``.ini`` style,
  however this function supports ``include`` directives that can include
  additional configuration files.

  ::

    [include "path/to/config.ini"]            ; errors if the file does not exist
    [include "path/to/config.ini" if-exists]  ; ignored if the file does not exist

  :param filename: The name of the configuration file to read.
  :param basedir: If *filename* is not an absolute path or the base directory
    should be altered, this is the directory of which to look for files
    specified with ``include`` directives.
  :param follow_include_directives: If this is True, ``include`` directives
    will be followed.
  :raise FileNotFoundError: If *filename* does not exist.
  :raise InvalidConfigError: If the configuration format is invalid. Also
    if any of the included files do not exist.
  :return: A dictionary. Section names are prepended to the option names.
  """

    filename = path.norm(filename)
    if not basedir:
        basedir = path.dirname(filename)

    if not path.isfile(filename):
        raise FileNotFoundError(filename)

    logger.debug("reading configuration file:", filename)
    parser = configparser.SafeConfigParser()
    try:
        parser.read([filename])
    except configparser.Error as exc:
        raise InvalidConfigError('"{}": {}'.format(filename, exc))

    result = {}
    for section in parser.sections():
        match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
        if match:
            if not follow_include_directives:
                continue
            ifile, if_exists = match.groups()
            ifile = path.norm(ifile, basedir)
            try:
                result.update(read_config_file(ifile))
            except FileNotFoundError as exc:
                if not if_exists:
                    raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename))
            continue
        elif section == "__global__":
            prefix = ""
        else:
            prefix = section + "."

        for option in parser.options(section):
            result[prefix + option] = parser.get(section, option)

    return result
Пример #8
0
  def execute(self, parser, args):
    session.path.extend(map(path.norm, args.include_path))

    if self.is_export:
      # Determine the module to execute, either from the current working
      # directory or find it by name if one is specified.
      if not args.module:
        for fn in [MANIFEST_FILENAME, path.join('craftr', MANIFEST_FILENAME)]:
          if path.isfile(fn):
            module = session.parse_manifest(fn)
            break
        else:
          parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
      else:
        # TODO: For some reason, prints to stdout are not visible here.
        # TODO: Prints to stderr however work fine.
        try:
          module_name, version = parse_module_spec(args.module)
        except ValueError as exc:
          parser.error('{} (note: you have to escape > and < characters)'.format(exc))
        try:
          module = session.find_module(module_name, version)
        except Module.NotFound as exc:
          parser.error('module not found: ' + str(exc))
    else:
      module = None

    ninja_bin, ninja_version = get_ninja_info()

    # Create and switch to the build directory.
    session.builddir = path.abs(args.build_dir)
    path.makedirs(session.builddir)
    os.chdir(session.builddir)

    # Read the cache and parse command-line options.
    cachefile = path.join(session.builddir, '.craftrcache')
    if not read_cache(cachefile) and not self.is_export:
      logger.error('Unable to load "{}", can not build'.format(cachefile))
      return 1

    # Prepare options, loaders and execute.
    if self.is_export:
      session.cache['build'] = {}
      try:
        write_cache(cachefile)
        module.run()
      except (Module.InvalidOption, Module.LoaderInitializationError) as exc:
        for error in exc.format_errors():
          logger.error(error)
        return 1
      except craftr.defaults.ModuleError as exc:
        logger.error(exc)
        return 1

      # Write the cache back.
      session.cache['build']['targets'] = list(session.graph.targets.keys())
      session.cache['build']['main'] = module.ident
      session.cache['build']['options'] = args.options
      write_cache(cachefile)

      # Write the Ninja manifest.
      with open("build.ninja", 'w') as fp:
        platform = core.build.get_platform_helper()
        context = core.build.ExportContext(ninja_version)
        writer = core.build.NinjaWriter(fp)
        session.graph.export(writer, context, platform)

    else:
      parse_cmdline_options(session.cache['build']['options'])
      main = session.cache['build']['main']
      available_targets = frozenset(session.cache['build']['targets'])

      # Check the targets and if they exist.
      targets = []
      for target in args.targets:
        if '.' not in target:
          target = main + '.' + target
        elif target.startswith('.'):
          target = main + target

        module_name, target = target.rpartition('.')[::2]
        module_name, version = get_volatile_module_version(module_name)
        ref_module = session.find_module(module_name, version or '*')
        target = craftr.targetbuilder.get_full_name(target, ref_module)
        if target not in available_targets:
          parser.error('no such target: {}'.format(target))
        targets.append(target)

      # Execute the ninja build.
      cmd = [ninja_bin]
      if args.verbose:
        cmd += ['-v']
      cmd += targets
      shell.run(cmd)
Пример #9
0
    def load(self, context, cache):
        if cache is not None and path.isdir(cache.get("directory", "")):
            # Check if the requested version changes.
            url_template = context.expand_variables(cache.get("url_template", ""))
            if url_template == cache.get("url"):
                self.directory = cache["directory"]
                logger.info("Reusing cached directory: {}".format(path.rel(self.directory, nopar=True)))
                return cache
            else:
                logger.info("Cached URL is outdated:", cache.get("url"))

        directory = None
        archive = None
        delete_after_extract = True
        for url_template in self.urls:
            url = context.expand_variables(url_template)
            if not url:
                continue
            if url.startswith("file://"):
                name = url[7:]
                if path.isdir(name):
                    logger.info("Using directory", url)
                    directory = name
                    break
                elif path.isfile(name):
                    logger.info("Using archive", url)
                    archive = name
                    delete_after_extract = False
                    break
                error = None
            else:
                error = None
                try:
                    progress = lambda d: self._download_progress(url, context, d)
                    archive, reused = httputils.download_file(
                        url, directory=context.get_temporary_directory(), on_exists="skip", progress=progress
                    )
                except (httputils.URLError, httputils.HTTPError) as exc:
                    error = exc
                except self.DownloadAlreadyExists as exc:
                    directory = exc.directory
                    logger.info("Reusing existing directory", directory)
                else:
                    if reused:
                        logger.info("Reusing cached download", path.basename(archive))
                    break

            if error:
                logger.info("Error reading", url, ":", error)

        if directory or archive:
            logger.debug("URL applies: {}".format(url))

        if not directory and archive:
            suffix, directory = self._get_archive_unpack_info(context, archive)
            logger.info(
                'Unpacking "{}" to "{}" ...'.format(path.rel(archive, nopar=True), path.rel(directory, nopar=True))
            )
            nr.misc.archive.extract(
                archive,
                directory,
                suffix=suffix,
                unpack_single_dir=True,
                check_extract_file=self._check_extract_file,
                progress_callback=self._extract_progress,
            )
        elif not directory:
            raise LoaderError(self, "no URL matched")

        self.directory = directory
        with open(path.join(self.directory, ".craftr_downloadurl"), "w") as fp:
            fp.write(url)
        return {"directory": directory, "url_template": url_template, "url": url}
Пример #10
0
    def execute(self, parser, args):
        session.path.extend(map(path.norm, args.include_path))

        if self.mode == "export":
            # Determine the module to execute, either from the current working
            # directory or find it by name if one is specified.
            if not args.module:
                for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]:
                    if path.isfile(fn):
                        module = session.parse_manifest(fn)
                        break
                else:
                    parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
            else:
                # TODO: For some reason, prints to stdout are not visible here.
                # TODO: Prints to stderr however work fine.
                try:
                    module_name, version = parse_module_spec(args.module)
                except ValueError as exc:
                    parser.error("{} (note: you have to escape > and < characters)".format(exc))
                try:
                    module = session.find_module(module_name, version)
                except Module.NotFound as exc:
                    parser.error("module not found: " + str(exc))
        else:
            module = None

        ninja_bin, ninja_version = get_ninja_info()

        # Create and switch to the build directory.
        session.builddir = path.abs(args.build_dir)
        path.makedirs(session.builddir)
        os.chdir(session.builddir)

        # Read the cache and parse command-line options.
        cachefile = path.join(session.builddir, ".craftrcache")
        if not read_cache(cachefile) and self.mode != "export":
            logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode))
            logger.error("Make sure to generate a build tree with 'craftr export'")
            return 1

        # Prepare options, loaders and execute.
        if self.mode == "export":
            session.expand_relative_options(module.manifest.name)
            session.cache["build"] = {}
            try:
                module.run()
            except Module.InvalidOption as exc:
                for error in exc.format_errors():
                    logger.error(error)
                return 1
            except craftr.defaults.ModuleError as exc:
                logger.error("error:", exc)
                return 1
            finally:
                if sys.exc_info():
                    # We still want to write the cache, especially so that data already
                    # loaded with loaders doesn't need to be re-loaded. They'll find out
                    # when the cached information was not valid.
                    write_cache(cachefile)

            # Write the cache back.
            session.cache["build"]["targets"] = list(session.graph.targets.keys())
            session.cache["build"]["main"] = module.ident
            session.cache["build"]["options"] = args.options
            write_cache(cachefile)

            # Write the Ninja manifest.
            with open("build.ninja", "w") as fp:
                platform = core.build.get_platform_helper()
                context = core.build.ExportContext(ninja_version)
                writer = core.build.NinjaWriter(fp)
                session.graph.export(writer, context, platform)

        else:
            parse_cmdline_options(session.cache["build"]["options"])
            main = session.cache["build"]["main"]
            available_targets = frozenset(session.cache["build"]["targets"])

            logger.debug("build main module:", main)
            session.expand_relative_options(get_volatile_module_version(main)[0])

            # Check the targets and if they exist.
            targets = []
            for target in args.targets:
                if "." not in target:
                    target = main + "." + target
                elif target.startswith("."):
                    target = main + target

                module_name, target = target.rpartition(".")[::2]
                module_name, version = get_volatile_module_version(module_name)
                ref_module = session.find_module(module_name, version or "*")
                target = craftr.targetbuilder.get_full_name(target, ref_module)
                if target not in available_targets:
                    parser.error("no such target: {}".format(target))
                targets.append(target)

            # Execute the ninja build.
            cmd = [ninja_bin]
            if args.verbose:
                cmd += ["-v"]
            if self.mode == "clean":
                cmd += ["-t", "clean"]
                if not args.recursive:
                    cmd += ["-r"]
            cmd += targets
            return shell.run(cmd).returncode
Пример #11
0
def read_config_file(filename, basedir=None, follow_include_directives=True):
    """
  Reads a configuration file and returns a dictionary of the values that
  it contains. The format is standard :mod:`configparser` ``.ini`` style,
  however this function supports ``include`` directives that can include
  additional configuration files.

  ::

    [include "path/to/config.ini"]            ; errors if the file does not exist
    [include "path/to/config.ini" if-exists]  ; ignored if the file does not exist

  :param filename: The name of the configuration file to read.
  :param basedir: If *filename* is not an absolute path or the base directory
    should be altered, this is the directory of which to look for files
    specified with ``include`` directives.
  :param follow_include_directives: If this is True, ``include`` directives
    will be followed.
  :raise FileNotFoundError: If *filename* does not exist.
  :raise InvalidConfigError: If the configuration format is invalid. Also
    if any of the included files do not exist.
  :return: A dictionary. Section names are prepended to the option names.
  """

    filename = path.norm(filename)
    if not basedir:
        basedir = path.dirname(filename)

    if not path.isfile(filename):
        raise FileNotFoundError(filename)

    logger.debug('reading configuration file:', filename)
    parser = configparser.SafeConfigParser()
    try:
        parser.read([filename])
    except configparser.Error as exc:
        raise InvalidConfigError('"{}": {}'.format(filename, exc))

    result = {}
    for section in parser.sections():
        match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
        if match:
            if not follow_include_directives:
                continue
            ifile, if_exists = match.groups()
            ifile = path.norm(ifile, basedir)
            try:
                result.update(read_config_file(ifile))
            except FileNotFoundError as exc:
                if not if_exists:
                    raise InvalidConfigError(
                        'file "{}" included by "{}" does not exist'.format(
                            str(exc), filename))
            continue
        elif section == '__global__':
            prefix = ''
        else:
            prefix = section + '.'

        for option in parser.options(section):
            result[prefix + option] = parser.get(section, option)

    return result