Пример #1
0
  def parse_manifest(self, filename):
    """
    Parse a manifest by filename and add register the module to the module
    cache. Returns the :class:`Module` object. If the manifest has already
    been parsed, it will not be re-parsed.

    :raise Manifest.Invalid: If the manifest is invalid.
    :return: :const:`None` if the manifest is a duplicate of an already
      parsed manifest (determined by name and version), otherwise the
      :class:`Module` object for the manifest's module.
    """

    filename = path.norm(path.abs(filename))
    if filename in self._manifest_cache:
      manifest = self._manifest_cache[filename]
      return self.find_module(manifest.name, manifest.version)

    manifest = Manifest.parse(filename)
    self._manifest_cache[filename] = manifest
    versions = self.modules.setdefault(manifest.name, {})
    if manifest.version in versions:
      logger.debug('multiple occurences of "{}-{}" found, '
          'one of which is located at "{}"'.format(manifest.name,
          manifest.version, filename))
      module = None
    else:
      logger.debug('parsed manifest: {}-{} ({})'.format(
          manifest.name, manifest.version, filename))
      module = Module(path.dirname(filename), manifest)
      versions[manifest.version] = module

    return module
Пример #2
0
  def write_command_file(self, filename, commands, inputs=None, outputs=None,
      cwd=None, environ=None, foreach=False, suffix='.cmd', dry=False,
      accept_additional_args=False):

    if suffix is not None:
      filename = path.addsuffix(filename, suffix)
    result = ['cmd', '/Q', '/c', filename]
    if foreach:
      result += ['$in', '$out']
      inputs, outputs = ['%1'], ['%2']

    commands = self.replace_commands_inout_vars(commands, inputs, outputs)
    if dry:
      return result, filename

    path.makedirs(path.dirname(path.abs(filename)))
    with open(filename, 'w') as fp:
      fp.write('REM This file is automatically generated with Craftr. It is \n')
      fp.write('REM not recommended to modify it manually.\n\n')
      if cwd is not None:
        fp.write('cd ' + shell.quote(cwd) + '\n\n')
      for key, value in environ.items():
        fp.write('set ' + shell.quote('{}={}'.format(key, value), for_ninja=True) + '\n')
      fp.write('\n')
      for index, command in enumerate(commands):
        if accept_additional_args and index == len(commands)-1:
          command.append(shell.safe('%*'))
        fp.write(shell.join(command) + '\n')
        fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n')

    return result, filename
Пример #3
0
    def parse_manifest(self, filename):
        """
    Parse a manifest by filename and add register the module to the module
    cache. Returns the :class:`Module` object. If the manifest has already
    been parsed, it will not be re-parsed.

    :raise Manifest.Invalid: If the manifest is invalid.
    :return: :const:`None` if the manifest is a duplicate of an already
      parsed manifest (determined by name and version), otherwise the
      :class:`Module` object for the manifest's module.
    """

        filename = path.norm(path.abs(filename))
        if filename in self._manifest_cache:
            manifest = self._manifest_cache[filename]
            return self.find_module(manifest.name, manifest.version)

        manifest = Manifest.parse(filename)
        self._manifest_cache[filename] = manifest
        versions = self.modules.setdefault(manifest.name, {})
        if manifest.version in versions:
            other = versions[manifest.version].manifest.filename
            logger.debug('multiple occurences of "{}-{}" found\n'
                         '  - {}\n  - {}'.format(manifest.name,
                                                 manifest.version, filename,
                                                 other))
            module = None
        else:
            logger.debug('parsed manifest: {}-{} ({})'.format(
                manifest.name, manifest.version, filename))
            module = Module(path.dirname(filename), manifest)
            versions[manifest.version] = module

        return module
Пример #4
0
def write_cache(cachefile):
    # Write back the cache.
    try:
        path.makedirs(path.dirname(cachefile))
        with open(cachefile, 'w') as fp:
            session.write_cache(fp)
    except OSError as exc:
        logger.error('error writing cache file:', cachefile)
        logger.error(exc, indent=1)
    else:
        logger.debug('cache written:', cachefile)
Пример #5
0
def write_cache(cachefile):
  # Write back the cache.
  try:
    path.makedirs(path.dirname(cachefile))
    with open(cachefile, 'w') as fp:
      session.write_cache(fp)
  except OSError as exc:
    logger.error('error writing cache file:', cachefile)
    logger.error(exc, indent=1)
  else:
    logger.debug('cache written:', cachefile)
Пример #6
0
    def write_command_file(self,
                           filename,
                           commands,
                           inputs=None,
                           outputs=None,
                           cwd=None,
                           environ=None,
                           foreach=False,
                           suffix='.sh',
                           dry=False,
                           accept_additional_args=False):

        if suffix is not None:
            filename = path.addsuffix(filename, suffix)
        result = [filename]
        if foreach:
            result += ['$in', '$out']
            inputs, outputs = ['%1'], ['%2']

        commands = self.replace_commands_inout_vars(commands, inputs, outputs)
        if dry:
            return result, filename

        path.makedirs(path.dirname(filename))
        with open(filename, 'w') as fp:
            # TODO: Make sure this also works for shells other than bash.
            fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) +
                     '\n')
            fp.write('set -e\n')
            if cwd:
                fp.write('cd ' + shell.quote(cwd) + '\n')
            fp.write('\n')
            for key, value in environ.items():
                fp.write('export {}={}\n'.format(key, shell.quote(value)))
            fp.write('\n')
            for index, command in enumerate(commands):
                if accept_additional_args and index == len(commands) - 1:
                    command.append(shell.safe('$*'))
                fp.write(shell.join(command))
                fp.write('\n')

        os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
                 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)  # rwxrw-r--

        return result, filename
Пример #7
0
    def write_command_file(self,
                           filename,
                           commands,
                           inputs=None,
                           outputs=None,
                           cwd=None,
                           environ=None,
                           foreach=False,
                           suffix='.cmd',
                           dry=False,
                           accept_additional_args=False):

        if suffix is not None:
            filename = path.addsuffix(filename, suffix)
        result = ['cmd', '/Q', '/c', filename]
        if foreach:
            result += ['$in', '$out']
            inputs, outputs = ['%1'], ['%2']

        commands = self.replace_commands_inout_vars(commands, inputs, outputs)
        if dry:
            return result, filename

        path.makedirs(path.dirname(path.abs(filename)))
        with open(filename, 'w') as fp:
            fp.write(
                'REM This file is automatically generated with Craftr. It is \n'
            )
            fp.write('REM not recommended to modify it manually.\n\n')
            if cwd is not None:
                fp.write('cd ' + shell.quote(cwd) + '\n\n')
            for key, value in environ.items():
                fp.write(
                    'set ' +
                    shell.quote('{}={}'.format(key, value), for_ninja=True) +
                    '\n')
            fp.write('\n')
            for index, command in enumerate(commands):
                if accept_additional_args and index == len(commands) - 1:
                    command.append(shell.safe('%*'))
                fp.write(shell.join(command) + '\n')
                fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n')

        return result, filename
Пример #8
0
def write_response_file(arguments, builder=None, name=None, force_file=False, suffix=''):
  """
  Creates a response-file with the specified *name* in the in the
  ``buildfiles/`` directory and writes the *arguments* list quoted into
  the file. If *builder* is specified, it must be a :class:`TargetBuilder`
  and the response file will be added to the implicit dependencies.

  If *force_file* is set to True, a file will always be written. Otherwise,
  the function will into possible limitations of the platform and decide
  whether to write a response file or to return the *arguments* as is.

  Returns a tuple of ``(filename, arguments)``. If a response file is written,
  the returned *arguments* will be a list with a single string that is the
  filename prepended with ``@``. The *filename* part can be None if no
  response file needed to be exported.
  """

  if not name:
    if not builder:
      raise ValueError('builder must be specified if name is bot')
    name = builder.name + suffix + '.response.txt'

  if platform.name != 'win':
    return None, arguments

  # We'll just assume that there won't be more than 2048 characters for
  # other flags. The windows max buffer size is 8192.
  content = shell.join(arguments)
  if len(content) < 6144:
    return None, arguments

  filename = buildlocal(path.join('buildfiles', name))
  if builder:
    builder.implicit_deps.append(filename)

  if session.builddir:
    path.makedirs(path.dirname(filename))
    with open(filename, 'w') as fp:
      fp.write(content)
  return filename, ['@' + filename]
Пример #9
0
def write_response_file(arguments, builder=None, name=None, force_file=False):
  """
  Creates a response-file with the specified *name* in the in the
  ``buildfiles/`` directory and writes the *arguments* list quoted into
  the file. If *builder* is specified, it must be a :class:`TargetBuilder`
  and the response file will be added to the implicit dependencies.

  If *force_file* is set to True, a file will always be written. Otherwise,
  the function will into possible limitations of the platform and decide
  whether to write a response file or to return the *arguments* as is.

  Returns a tuple of ``(filename, arguments)``. If a response file is written,
  the returned *arguments* will be a list with a single string that is the
  filename prepended with ``@``. The *filename* part can be None if no
  response file needed to be exported.
  """

  if not name:
    if not builder:
      raise ValueError('builder must be specified if name is bot')
    name = builder.name + '.response.txt'

  if platform.name != 'win':
    return None, arguments

  # We'll just assume that there won't be more than 2048 characters for
  # other flags. The windows max buffer size is 8192.
  content = shell.join(arguments)
  if len(content) < 6144:
    return None, arguments

  filename = buildlocal(path.join('buildfiles', name))
  if builder:
    builder.implicit_deps.append(filename)

  if session.builddir:
    path.makedirs(path.dirname(filename))
    with open(filename, 'w') as fp:
      fp.write(content)
  return filename, ['@' + filename]
Пример #10
0
  def write_command_file(self, filename, commands, inputs=None, outputs=None,
      cwd=None, environ=None, foreach=False, suffix='.sh', dry=False,
      accept_additional_args=False):

    if suffix is not None:
      filename = path.addsuffix(filename, suffix)
    result = [filename]
    if foreach:
      result += ['$in', '$out']
      inputs, outputs = ['%1'], ['%2']

    commands = self.replace_commands_inout_vars(commands, inputs, outputs)
    if dry:
      return result, filename

    path.makedirs(path.dirname(filename))
    with open(filename, 'w') as fp:
      # TODO: Make sure this also works for shells other than bash.
      fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) + '\n')
      fp.write('set -e\n')
      if cwd:
        fp.write('cd ' + shell.quote(cwd) + '\n')
      fp.write('\n')
      for key, value in environ.items():
        fp.write('export {}={}\n'.format(key, shell.quote(value)))
      fp.write('\n')
      for index, command in enumerate(commands):
        if accept_additional_args and index == len(commands)-1:
          command.append(shell.safe('$*'))
        fp.write(shell.join(command))
        fp.write('\n')

    os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
      stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)  # rwxrw-r--

    return result, filename
Пример #11
0
    def _export_run_or_help(self, args, module):
        """
    Called when the mode is 'export' or 'run'. Will execute the specified
    *module* and eventually export a Ninja manifest and Cache.
    """

        read_cache(False)

        session.expand_relative_options()
        session.cache['build'] = {}

        # Load the dependency lock information if it exists.
        deplock_fn = path.join(path.dirname(module.manifest.filename),
                               '.dependency-lock')
        if os.path.isfile(deplock_fn):
            with open(deplock_fn) as fp:
                session.preferred_versions = cson.load(fp)
                logger.debug('note: dependency lock file "{}" loaded'.format(
                    deplock_fn))

        try:
            module.run()
        except Module.InvalidOption as exc:
            for error in exc.format_errors():
                logger.error(error)
            return 1
        except craftr.defaults.ModuleError as exc:
            logger.error('error:', exc)
            return 1
        finally:
            if sys.exc_info() and self.mode == 'export':
                # We still want to write the cache, especially so that data already
                # loaded with loaders doesn't need to be re-loaded. They'll find out
                # when the cached information was not valid.
                write_cache(self.cachefile)

        # Fill the cache.
        session.cache['build']['targets'] = list(session.graph.targets.keys())
        session.cache['build']['modules'] = serialise_loaded_module_info()
        session.cache['build']['main'] = module.ident
        session.cache['build']['options'] = args.options
        session.cache['build']['dependency_lock_filename'] = deplock_fn

        if self.mode == 'export':
            # Add the Craftr_run_command variable which is necessary for tasks
            # to properly executed.
            run_command = ['craftr', '-q', '-P', path.rel(session.maindir)]
            if args.no_config: run_command += ['-C']
            run_command += ['-c' + x for x in args.config]
            run_command += ['run']
            if args.module: run_command += ['-m', args.module]
            run_command += ['-i' + x for x in args.include_path]
            run_command += ['-b', path.rel(session.builddir)]
            session.graph.vars['Craftr_run_command'] = shell.join(run_command)

            write_cache(self.cachefile)

            # Write the Ninja manifest.
            with open("build.ninja", 'w') as fp:
                platform = core.build.get_platform_helper()
                context = core.build.ExportContext(self.ninja_version)
                writer = core.build.NinjaWriter(fp)
                session.graph.export(writer, context, platform)
                logger.info('exported "build.ninja"')

            return 0

        elif self.mode == 'run':
            if args.task:
                if args.task not in session.graph.tasks:
                    logger.error('no such task exists: "{}"'.format(args.task))
                    return 1
                task = session.graph.tasks[args.task]
                return task.invoke(args.task_args)
            return 0

        elif self.mode == 'help':
            if args.name not in vars(module.namespace):
                logger.error('symbol not found: "{}:{}"'.format(
                    module.manifest.name, args.name))
                return 1
            help(getattr(module.namespace, args.name))
            return 0

        assert False, "unhandled mode: {}".format(self.mode)
Пример #12
0
def read_config_file(filename, basedir=None, follow_include_directives=True):
    """
  Reads a configuration file and returns a dictionary of the values that
  it contains. The format is standard :mod:`configparser` ``.ini`` style,
  however this function supports ``include`` directives that can include
  additional configuration files.

  ::

    [include "path/to/config.ini"]            ; errors if the file does not exist
    [include "path/to/config.ini" if-exists]  ; ignored if the file does not exist

  :param filename: The name of the configuration file to read.
  :param basedir: If *filename* is not an absolute path or the base directory
    should be altered, this is the directory of which to look for files
    specified with ``include`` directives.
  :param follow_include_directives: If this is True, ``include`` directives
    will be followed.
  :raise FileNotFoundError: If *filename* does not exist.
  :raise InvalidConfigError: If the configuration format is invalid. Also
    if any of the included files do not exist.
  :return: A dictionary. Section names are prepended to the option names.
  """

    filename = path.norm(filename)
    if not basedir:
        basedir = path.dirname(filename)

    if not path.isfile(filename):
        raise FileNotFoundError(filename)

    logger.debug("reading configuration file:", filename)
    parser = configparser.SafeConfigParser()
    try:
        parser.read([filename])
    except configparser.Error as exc:
        raise InvalidConfigError('"{}": {}'.format(filename, exc))

    result = {}
    for section in parser.sections():
        match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
        if match:
            if not follow_include_directives:
                continue
            ifile, if_exists = match.groups()
            ifile = path.norm(ifile, basedir)
            try:
                result.update(read_config_file(ifile))
            except FileNotFoundError as exc:
                if not if_exists:
                    raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename))
            continue
        elif section == "__global__":
            prefix = ""
        else:
            prefix = section + "."

        for option in parser.options(section):
            result[prefix + option] = parser.get(section, option)

    return result
Пример #13
0
def download_file(url,
                  filename=None,
                  file=None,
                  directory=None,
                  on_exists='rename',
                  progress=None,
                  chunksize=4096,
                  urlopen_kwargs=None):
    """
  Download a file from a URL to one of the following destinations:

  :param filename: A filename to write the downloaded file to.
  :param file: A file-like object.
  :param directory: A directory. The filename will be automatically
    determined from the ``Content-Disposition`` header received from
    the server or the last path elemnet in the URL.

  Additional parameters for the *directory* parameter:

  :param on_exists: The operation to perform when the file already exists.
    Available modes are ``rename``, ``overwrite`` and ``skip``.

  Additional parameters:

  :param progress: A callable that accepts a single parameter that is a
    dictionary with information about the progress of the download. The
    dictionary provides the keys ``size``,  ``downloaded`` and ``response``.
    If the callable returns :const:`False` (specifically the value False), the
    download will be aborted and a :class:`UserInterrupt` will be raised.
  :param urlopen_kwargs: A dictionary with additional keyword arguments
    for :func:`urllib.request.urlopen`.

  Raise and return:

  :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
  :raise URLError: Can be raised by :func:`urllib.request.urlopen`.
  :raise UserInterrupt: If the *progress* returned :const:`False`.
  :return: If the download mode is *directory*, the name of the downloaded
    file will be returned and False if the file was newly downloaded, True
    if the download was skipped because the file already existed.
    Otherwise, the number of bytes downloaded will be returned.
  """

    argspec.validate('on_exists', on_exists,
                     {'enum': ['rename', 'overwrite', 'skip']})

    if sum(map(bool, [filename, file, directory])) != 1:
        raise ValueError(
            'exactly one of filename, file or directory must be specifed')

    response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))

    if directory:
        try:
            filename = parse_content_disposition(
                response.headers.get('Content-Disposition', ''))
        except ValueError:
            filename = url.split('/')[-1]
        filename = path.join(directory, filename)
        path.makedirs(directory)

        if path.exists(filename):
            if on_exists == 'skip':
                return filename, True
            elif on_exists == 'rename':
                index = 0
                while True:
                    new_filename = filename + '_{:0>4}'.format(index)
                    if not path.exists(new_filename):
                        filename = new_filename
                        break
                    index += 1
            elif on_exists != 'overwrite':
                raise RuntimeError

    try:
        size = int(response.headers.get('Content-Length', ''))
    except ValueError:
        size = None

    progress_info = {
        'response': response,
        'size': size,
        'downloaded': 0,
        'completed': False,
        'filename': filename,
        'url': url
    }
    if progress and progress(progress_info) is False:
        raise UserInterrupt

    def copy_to_file(fp):
        while True:
            data = response.read(chunksize)
            if not data:
                break
            progress_info['downloaded'] += len(data)
            fp.write(data)
            if progress and progress(progress_info) is False:
                raise UserInterrupt
        progress_info['completed'] = True
        if progress and progress(progress_info) is False:
            raise UserInterrupt

    if filename:
        path.makedirs(path.dirname(filename))
        try:
            with open(filename, 'wb') as fp:
                copy_to_file(fp)
        except BaseException:
            # Delete the file if it could not be downloaded successfully.
            path.remove(filename, silent=True)
            raise
    elif file:
        copy_to_file(file)

    return filename, False
Пример #14
0
def download_file(url, filename=None, file=None, directory=None,
    on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None):
  """
  Download a file from a URL to one of the following destinations:

  :param filename: A filename to write the downloaded file to.
  :param file: A file-like object.
  :param directory: A directory. The filename will be automatically
    determined from the ``Content-Disposition`` header received from
    the server or the last path elemnet in the URL.

  Additional parameters for the *directory* parameter:

  :param on_exists: The operation to perform when the file already exists.
    Available modes are ``rename``, ``overwrite`` and ``skip``.

  Additional parameters:

  :param progress: A callable that accepts a single parameter that is a
    dictionary with information about the progress of the download. The
    dictionary provides the keys ``size``,  ``downloaded`` and ``response``.
    If the callable returns :const:`False` (specifically the value False), the
    download will be aborted and a :class:`UserInterrupt` will be raised.
  :param urlopen_kwargs: A dictionary with additional keyword arguments
    for :func:`urllib.request.urlopen`.

  Raise and return:

  :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
  :raise URLError: Can be raised by :func:`urllib.request.urlopen`.
  :raise UserInterrupt: If the *progress* returned :const:`False`.
  :return: If the download mode is *directory*, the name of the downloaded
    file will be returned and False if the file was newly downloaded, True
    if the download was skipped because the file already existed.
    Otherwise, the number of bytes downloaded will be returned.
  """

  argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']})

  if sum(map(bool, [filename, file, directory])) != 1:
    raise ValueError('exactly one of filename, file or directory must be specifed')

  response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))

  if directory:
    try:
      filename = parse_content_disposition(
        response.headers.get('Content-Disposition', ''))
    except ValueError:
      filename = url.split('/')[-1]
    filename = path.join(directory, filename)
    path.makedirs(directory)

    if path.exists(filename):
      if on_exists == 'skip':
        return filename, True
      elif on_exists == 'rename':
        index = 0
        while True:
          new_filename = filename + '_{:0>4}'.format(index)
          if not path.exists(new_filename):
            filename = new_filename
            break
          index += 1
      elif on_exists != 'overwrite':
        raise RuntimeError

  try:
    size = int(response.headers.get('Content-Length', ''))
  except ValueError:
    size = None

  progress_info = {'response': response, 'size': size, 'downloaded': 0,
    'completed': False, 'filename': filename, 'url': url}
  if progress and progress(progress_info) is False:
    raise UserInterrupt

  def copy_to_file(fp):
    while True:
      data = response.read(chunksize)
      if not data:
        break
      progress_info['downloaded'] += len(data)
      fp.write(data)
      if progress and progress(progress_info) is False:
        raise UserInterrupt
    progress_info['completed'] = True
    if progress and progress(progress_info) is False:
      raise UserInterrupt

  if filename:
    path.makedirs(path.dirname(filename))
    try:
      with open(filename, 'wb') as fp:
        copy_to_file(fp)
    except BaseException:
      # Delete the file if it could not be downloaded successfully.
      path.remove(filename, silent=True)
      raise
  elif file:
    copy_to_file(file)

  return filename, False
Пример #15
0
def read_config_file(filename, basedir=None, follow_include_directives=True):
    """
  Reads a configuration file and returns a dictionary of the values that
  it contains. The format is standard :mod:`configparser` ``.ini`` style,
  however this function supports ``include`` directives that can include
  additional configuration files.

  ::

    [include "path/to/config.ini"]            ; errors if the file does not exist
    [include "path/to/config.ini" if-exists]  ; ignored if the file does not exist

  :param filename: The name of the configuration file to read.
  :param basedir: If *filename* is not an absolute path or the base directory
    should be altered, this is the directory of which to look for files
    specified with ``include`` directives.
  :param follow_include_directives: If this is True, ``include`` directives
    will be followed.
  :raise FileNotFoundError: If *filename* does not exist.
  :raise InvalidConfigError: If the configuration format is invalid. Also
    if any of the included files do not exist.
  :return: A dictionary. Section names are prepended to the option names.
  """

    filename = path.norm(filename)
    if not basedir:
        basedir = path.dirname(filename)

    if not path.isfile(filename):
        raise FileNotFoundError(filename)

    logger.debug('reading configuration file:', filename)
    parser = configparser.SafeConfigParser()
    try:
        parser.read([filename])
    except configparser.Error as exc:
        raise InvalidConfigError('"{}": {}'.format(filename, exc))

    result = {}
    for section in parser.sections():
        match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
        if match:
            if not follow_include_directives:
                continue
            ifile, if_exists = match.groups()
            ifile = path.norm(ifile, basedir)
            try:
                result.update(read_config_file(ifile))
            except FileNotFoundError as exc:
                if not if_exists:
                    raise InvalidConfigError(
                        'file "{}" included by "{}" does not exist'.format(
                            str(exc), filename))
            continue
        elif section == '__global__':
            prefix = ''
        else:
            prefix = section + '.'

        for option in parser.options(section):
            result[prefix + option] = parser.get(section, option)

    return result