Example #1
0
  def find_module(self, name, version):
    """
    Finds a module in the :attr:`path` matching the specified *name* and
    *version*.

    :param name: The name of the module.
    :param version: A :class:`VersionCriteria`, :class:`Version` or string
      in a VersionCritiera format.
    :raise ModuleNotFound: If the module can not be found.
    :return: :class:`Module`
    """

    argspec.validate('name', name, {'type': str})
    argspec.validate('version', version, {'type': [str, Version, VersionCriteria]})

    if isinstance(version, str):
      try:
        version = Version(version)
      except ValueError as exc:
        version = VersionCriteria(version)

    self.update_manifest_cache()
    if name in self.modules:
      if isinstance(version, Version):
        if version in self.modules[name]:
          return self.modules[name][version]
        raise ModuleNotFound(name, version)
      for module in sorted(self.modules[name].values(),
          key=lambda x: x.manifest.version, reverse=True):
        if version(module.manifest.version):
          return module

    raise ModuleNotFound(name, version)
Example #2
0
  def export(self, writer, context, platform):
    """
    Export the build graph to a Ninja manifest.

    :param writer: A :class:`ninja_syntax.Writer` object.
    :param context: A :class:`ExportContext` object.
    :param platform: A :class:`PlatformHelper` instance.
    """

    argspec.validate('writer', writer, {"type": ninja_syntax.Writer})
    writer.comment('This file was automatically generated with Craftr.')
    writer.comment('It is not recommended to edit this file manually.')
    writer.newline()

    if self.vars:
      for key, value in self.vars.items():
        writer.variable(key, value)
      writer.newline()

    if self.tools:
      writer.comment('Tools')
      writer.comment('-----')
      for tool in self.tools.values():
        tool.export(writer, context, platform)
      writer.newline()

    defaults = []
    for target in self.targets.values():
      if not target.explicit:
        defaults.append(target.name)
      target.export(writer, context, platform)

    if defaults:
      writer.default(defaults)
Example #3
0
    def export(self, writer, context, platform):
        """
    Export the build graph to a Ninja manifest.

    :param writer: A :class:`ninja_syntax.Writer` object.
    :param context: A :class:`ExportContext` object.
    :param platform: A :class:`PlatformHelper` instance.
    """

        argspec.validate('writer', writer, {"type": ninja_syntax.Writer})
        writer.comment('This file was automatically generated with Craftr.')
        writer.comment('It is not recommended to edit this file manually.')
        writer.newline()

        if self.vars:
            for key, value in self.vars.items():
                writer.variable(key, value)
            writer.newline()

        if self.tools:
            writer.comment('Tools')
            writer.comment('-----')
            for tool in self.tools.values():
                tool.export(writer, context, platform)
            writer.newline()

        defaults = []
        for target in self.targets.values():
            if not target.explicit and target.generates_build_instruction:
                defaults.append(target.name)
            target.export(writer, context, platform)

        if defaults:
            writer.default(defaults)
Example #4
0
    def find_module(self, name, version, resolve_preferred_version=True):
        """
    Finds a module in the :attr:`path` matching the specified *name* and
    *version*.

    :param name: The name of the module.
    :param version: A :class:`VersionCriteria`, :class:`Version` or string
      in a VersionCritiera format.
    :param resolve_preferred_version: If this parameter is True (default)
      and a preferred version is specified in :attr:`preferred_versions`,
      that preferred version is loaded or :class:`ModuleNotFound` is raised.
    :raise ModuleNotFound: If the module can not be found.
    :return: :class:`Module`
    """

        argspec.validate('name', name, {'type': str})
        argspec.validate('version', version,
                         {'type': [str, Version, VersionCriteria]})

        if name in renames.renames:
            logger.warn('"{}" is deprecated, use "{}" instead'.format(
                name, renames.renames[name]))
            name = renames.renames[name]

        if isinstance(version, str):
            try:
                version = Version(version)
            except ValueError as exc:
                version = VersionCriteria(version)

        if session.module and resolve_preferred_version:
            data = self.preferred_versions.get(session.module.manifest.name)
            if data is not None:
                versions = data.get(str(session.module.manifest.version))
                if versions is not None:
                    preferred_version = versions.get(name)
                    if preferred_version is not None:
                        version = Version(preferred_version)
                        logger.debug(
                            'note: loading preferred version {} of module "{}" '
                            'requested by module "{}"'.format(
                                version, name, session.module.ident))

        self.update_manifest_cache()
        if name in self.modules:
            if isinstance(version, Version):
                if version in self.modules[name]:
                    return self.modules[name][version]
                raise ModuleNotFound(name, version)
            for module in sorted(self.modules[name].values(),
                                 key=lambda x: x.manifest.version,
                                 reverse=True):
                if version(module.manifest.version):
                    return module

        raise ModuleNotFound(name, version)
Example #5
0
    def __init__(self, name, command, preamble=None, environ=None):
        argspec.validate('name', name, {'type': str})
        argspec.validate('command', command, {
            'type': [list, tuple],
            'allowEmpty': False,
            'items': {
                'type': str
            }
        })
        argspec.validate(
            'preamble', preamble, {
                'type': [None, list, tuple],
                'items': {
                    'type': [list, tuple],
                    'allowEmpty': False,
                    'items': {
                        'type': str
                    }
                }
            })
        argspec.validate('environ', environ, {'type': [None, dict]})

        self.name = name
        self.command = list(command)
        self.preamble = preamble or []
        self.environ = environ or {}
        self.exported_command = None
Example #6
0
    def add_task(self, task, **kwargs):
        """
    Add a :class:`Task` to the Graph. Creates a :class:`Target` for the task
    and returns it.

    :param task: The :class:`Task` to add to the Graph.
    :param kwargs: Additional parameters for the :class:`Target` constructor.
    """

        argspec.validate('task', task, {'type': Task})
        if task.name in self.tasks:
            raise ValueError('a task with the name {!r} already exists'.format(
                task.name))

        target = Target(task.name, [task.get_command()], task=task, **kwargs)
        assert target.task is task
        self.add_target(target)
        self.tasks[task.name] = task
        return target
Example #7
0
  def __init__(self, name, command, preamble=None, environ=None):
    argspec.validate('name', name, {'type': str})
    argspec.validate('command', command,
        {'type': [list, tuple], 'allowEmpty': False, 'items': {'type': str}})
    argspec.validate('preamble', preamble,
        {'type': [None, list, tuple], 'items':
          {'type': [list, tuple], 'allowEmpty': False, 'items': {'type': str}}})
    argspec.validate('environ', environ, {'type': [None, dict]})

    self.name = name
    self.command = command
    self.preamble = preamble or []
    self.environ = environ or {}
    self.exported_command = None
Example #8
0
  def add_target(self, target):
    """
    Add a :class:`Target` to the Graph.

    .. note:: For performance reasons, this method assumes that all paths
              in the *target* are already normalized with :meth:`path.norm`.

    :raise ValueError: If the :attr:`Target.name` is already used.
    :raise DuplicateOutputError: If the *target* lists an output file that
      is already created by another target.
    """

    argspec.validate('target', target, {'type': Target})
    if target.name in self.targets:
      raise ValueError('a target with the name {!r} already exists'
        .format(target.name))
    self.targets[target.name] = target
    for infile in target.inputs:
      self.infiles.setdefault(infile, []).append(target)
    for outfile in target.outputs:
      other = self.outfiles.setdefault(outfile, target)
      if other is not target:
        raise DuplicateOutputError(outfile, target, other)
Example #9
0
    def add_target(self, target):
        """
    Add a :class:`Target` to the Graph.

    .. note:: For performance reasons, this method assumes that all paths
              in the *target* are already normalized with :meth:`path.norm`.

    :raise ValueError: If the :attr:`Target.name` is already used.
    :raise DuplicateOutputError: If the *target* lists an output file that
      is already created by another target.
    """

        argspec.validate('target', target, {'type': Target})
        if target.name in self.targets:
            raise ValueError(
                'a target with the name {!r} already exists'.format(
                    target.name))
        self.targets[target.name] = target
        for infile in target.inputs:
            self.infiles.setdefault(infile, []).append(target)
        for outfile in target.outputs:
            other = self.outfiles.setdefault(outfile, target)
            if other is not target:
                raise DuplicateOutputError(outfile, target, other)
Example #10
0
def glob(patterns, parent=None, excludes=(), include_dotfiles=False):
    """
  Wrapper for :func:`glob2.glob` that accepts an arbitrary number of
  patterns and matches them. The paths are normalized with :func:`norm`.

  Relative patterns are automaticlly joined with *parent*. If the
  parameter is omitted, it defaults to the currently executed build
  scripts project directory.

  If *excludes* is specified, it must be a string or a list of strings
  that is/contains glob patterns or filenames to be removed from the
  result before returning.

  .. note::

    Every file listed in *excludes* will only remove **one** item from
    the result list that was generated from *patterns*. Thus, if you
    want to exclude some files with a pattern except for a specific file
    that would also match that pattern, simply list that file another
    time in the *patterns*.

  :param patterns: A list of glob patterns or filenames.
  :param parent: The parent directory for relative paths.
  :param excludes: A list of glob patterns or filenames.
  :param include_dotfiles: If True, ``*`` and ``**`` can also capture
    file or directory names starting with a dot.
  :return: A list of filenames.
  """

    argspec.validate("patterns", patterns, {"type": [list, tuple]})
    argspec.validate("excludes", excludes, {"type": [list, tuple]})
    argspec.validate("parent", parent, {"type": [None, str]})

    if not parent:
        parent = getcwd()

    result = []
    for pattern in patterns:
        if not isabs(pattern):
            pattern = join(parent, pattern)
        result += glob2.glob(norm(pattern))

    for pattern in excludes:
        if not isabs(pattern):
            pattern = join(parent, pattern)
        pattern = norm(pattern)
        if not isglob(pattern):
            result.remove(pattern)
        else:
            for item in glob2.glob(pattern):
                result.remove(item)

    return result
Example #11
0
  def __init__(self, name, commands, inputs, outputs, implicit_deps=(),
               order_only_deps=(), pool=None, deps=None, depfile=None,
               msvc_deps_prefix=None, explicit=False, foreach=False,
               description=None, metadata=None, cwd=None, environ=None,
               frameworks=()):
    argspec.validate('name', name, {'type': str})
    argspec.validate('commands', commands,
      {'type': list, 'allowEmpty': False, 'items':
        {'type': list, 'allowEmpty': False, 'items': {'type': [Tool, Target, str]}}})
    argspec.validate('inputs', inputs, {'type': [list, tuple], 'items': {'type': [Target, str]}})
    argspec.validate('outputs', outputs, {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('implicit_deps', implicit_deps, {'type': [list, tuple], 'items': {'type': [Target, str]}})
    argspec.validate('order_only_deps', order_only_deps, {'type': [list, tuple], 'items': {'type': [Target, str]}})
    argspec.validate('pool', pool, {'type': [None, str]})
    argspec.validate('deps', deps, {'type': [None, str], 'enum': ['msvc', 'gcc']})
    argspec.validate('depfile', depfile, {'type': [None, str]})
    argspec.validate('msvc_deps_prefix', msvc_deps_prefix, {'type': [None, str]})
    argspec.validate('explicit', explicit, {'type': bool})
    argspec.validate('foreach', foreach, {'type': bool})
    argspec.validate('description', description, {'type': [None, str]})
    argspec.validate('metadata', metadata, {'type': [None, dict]})
    argspec.validate('cwd', cwd, {'type': [None, str]})
    argspec.validate('environ', environ, {'type': [None, dict]})
    argspec.validate('frameworks', frameworks, {'type': [list, tuple], 'items': {'type': dict}})

    def expand_mixed_list(mixed, implicit_deps, mode):
      result = []
      for item in mixed:
        if isinstance(item, Target):
          if mode == 'implicit':
            names = [item.name]
          elif mode == 'inputs':
            names = item.outputs
          elif mode == 'cmd':
            names = [path.abs(x) for x in item.outputs]
          else:
            raise RuntimeError(mode)

          if implicit_deps is not None:
            implicit_deps += names
          result += names

        elif isinstance(item, str):
          if mode != 'cmd':
            item = path.abs(item)
          result.append(item)
        else: raise RuntimeError

      return result

    self.implicit_deps = []
    self.inputs = expand_mixed_list(inputs, None, 'inputs')
    self.outputs = [path.abs(x) for x in outputs]
    self.commands = [expand_mixed_list(cmd, self.implicit_deps, 'cmd') for cmd in commands]
    self.implicit_deps += expand_mixed_list(implicit_deps, None, 'implicit')
    self.order_only_deps = expand_mixed_list(order_only_deps, None, 'implicit')

    self.name = name
    self.pool = pool
    self.deps = deps
    self.depfile = depfile
    self.msvc_deps_prefix = msvc_deps_prefix
    self.explicit = explicit
    self.foreach = foreach
    self.description = description
    self.metadata = metadata or {}
    self.cwd = cwd
    self.environ = environ or {}
    self.frameworks = frameworks

    if self.foreach and len(self.inputs) != len(self.outputs):
      raise ValueError('foreach target must have the same number of output '
        'files ({}) as input files ({})'.format(len(self.outputs),
        len(self.inputs)))

    if self.deps == 'gcc' and not self.depfile:
      raise ValueError('require depfile with deps="gcc"')
Example #12
0
def download_file(url,
                  filename=None,
                  file=None,
                  directory=None,
                  on_exists='rename',
                  progress=None,
                  chunksize=4096,
                  urlopen_kwargs=None):
    """
  Download a file from a URL to one of the following destinations:

  :param filename: A filename to write the downloaded file to.
  :param file: A file-like object.
  :param directory: A directory. The filename will be automatically
    determined from the ``Content-Disposition`` header received from
    the server or the last path elemnet in the URL.

  Additional parameters for the *directory* parameter:

  :param on_exists: The operation to perform when the file already exists.
    Available modes are ``rename``, ``overwrite`` and ``skip``.

  Additional parameters:

  :param progress: A callable that accepts a single parameter that is a
    dictionary with information about the progress of the download. The
    dictionary provides the keys ``size``,  ``downloaded`` and ``response``.
    If the callable returns :const:`False` (specifically the value False), the
    download will be aborted and a :class:`UserInterrupt` will be raised.
  :param urlopen_kwargs: A dictionary with additional keyword arguments
    for :func:`urllib.request.urlopen`.

  Raise and return:

  :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
  :raise URLError: Can be raised by :func:`urllib.request.urlopen`.
  :raise UserInterrupt: If the *progress* returned :const:`False`.
  :return: If the download mode is *directory*, the name of the downloaded
    file will be returned and False if the file was newly downloaded, True
    if the download was skipped because the file already existed.
    Otherwise, the number of bytes downloaded will be returned.
  """

    argspec.validate('on_exists', on_exists,
                     {'enum': ['rename', 'overwrite', 'skip']})

    if sum(map(bool, [filename, file, directory])) != 1:
        raise ValueError(
            'exactly one of filename, file or directory must be specifed')

    response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))

    if directory:
        try:
            filename = parse_content_disposition(
                response.headers.get('Content-Disposition', ''))
        except ValueError:
            filename = url.split('/')[-1]
        filename = path.join(directory, filename)
        path.makedirs(directory)

        if path.exists(filename):
            if on_exists == 'skip':
                return filename, True
            elif on_exists == 'rename':
                index = 0
                while True:
                    new_filename = filename + '_{:0>4}'.format(index)
                    if not path.exists(new_filename):
                        filename = new_filename
                        break
                    index += 1
            elif on_exists != 'overwrite':
                raise RuntimeError

    try:
        size = int(response.headers.get('Content-Length', ''))
    except ValueError:
        size = None

    progress_info = {
        'response': response,
        'size': size,
        'downloaded': 0,
        'completed': False,
        'filename': filename,
        'url': url
    }
    if progress and progress(progress_info) is False:
        raise UserInterrupt

    def copy_to_file(fp):
        while True:
            data = response.read(chunksize)
            if not data:
                break
            progress_info['downloaded'] += len(data)
            fp.write(data)
            if progress and progress(progress_info) is False:
                raise UserInterrupt
        progress_info['completed'] = True
        if progress and progress(progress_info) is False:
            raise UserInterrupt

    if filename:
        path.makedirs(path.dirname(filename))
        try:
            with open(filename, 'wb') as fp:
                copy_to_file(fp)
        except BaseException:
            # Delete the file if it could not be downloaded successfully.
            path.remove(filename, silent=True)
            raise
    elif file:
        copy_to_file(file)

    return filename, False
Example #13
0
    def __init__(self,
                 name,
                 option_kwargs=None,
                 frameworks=(),
                 inputs=(),
                 outputs=(),
                 implicit_deps=(),
                 order_only_deps=()):
        argspec.validate('name', name, {'type': str})
        argspec.validate('option_kwargs', option_kwargs,
                         {'type': [None, dict, Framework]})
        argspec.validate('frameworks', frameworks, {
            'type': [list, tuple],
            'items': {
                'type': [Framework, build.Target]
            }
        })
        argspec.validate(
            'inputs', inputs, {
                'type': [list, tuple, build.Target],
                'items': {
                    'type': [str, build.Target]
                }
            })
        argspec.validate('outputs', outputs, {
            'type': [list, tuple],
            'items': {
                'type': str
            }
        })
        argspec.validate('implicit_deps', implicit_deps, {
            'type': [list, tuple],
            'items': {
                'type': str
            }
        })
        argspec.validate('order_only_deps', order_only_deps, {
            'type': [list, tuple],
            'items': {
                'type': str
            }
        })

        if isinstance(inputs, build.Target):
            inputs = [inputs]

        self.frameworks = []
        self.implicit_deps = list(implicit_deps)
        self.order_only_deps = list(order_only_deps)

        # If we find any Target objects in the inputs, expand the outputs
        # and append the frameworks.
        if inputs is not None:
            self.inputs = []
            for input_ in (inputs or ()):
                if isinstance(input_, build.Target):
                    pyutils.unique_extend(self.frameworks, input_.frameworks)
                    self.inputs += input_.outputs
                else:
                    self.inputs.append(input_)
        else:
            self.inputs = None

        # Unpack the frameworks list, which may also container Targets.
        for fw in frameworks:
            if isinstance(fw, build.Target):
                self.implicit_deps.append(fw)
                self.frameworks += fw.frameworks
            else:
                self.frameworks.append(fw)

        if option_kwargs is None:
            option_kwargs = {}

        self.name = name
        self.outputs = list(outputs)
        self.metadata = {}
        self.used_option_keys = set()

        self.option_kwargs = Framework(name, **option_kwargs)
        self.option_kwargs_defaults = Framework(name + "_defaults")
        self.options_merge = OptionMerge(self.option_kwargs,
                                         self.option_kwargs_defaults,
                                         *self.frameworks)
        assert self.option_kwargs in self.options_merge.frameworks
Example #14
0
  def __init__(self, name, commands, inputs, outputs, implicit_deps=(),
               order_only_deps=(), pool=None, deps=None, depfile=None,
               msvc_deps_prefix=None, explicit=False, foreach=False,
               description=None, metadata=None, cwd=None, environ=None,
               frameworks=()):
    argspec.validate('name', name, {'type': str})
    argspec.validate('commands', commands,
      {'type': list, 'allowEmpty': False, 'items':
        {'type': list, 'allowEmpty': False, 'items': {'type': [Tool, Target, str]}}})
    argspec.validate('inputs', inputs, {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('outputs', outputs, {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('implicit_deps', implicit_deps, {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('order_only_deps', order_only_deps, {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('pool', pool, {'type': [None, str]})
    argspec.validate('deps', deps, {'type': [None, str], 'enum': ['msvc', 'gcc']})
    argspec.validate('depfile', depfile, {'type': [None, str]})
    argspec.validate('msvc_deps_prefix', msvc_deps_prefix, {'type': [None, str]})
    argspec.validate('explicit', explicit, {'type': bool})
    argspec.validate('foreach', foreach, {'type': bool})
    argspec.validate('description', description, {'type': [None, str]})
    argspec.validate('metadata', metadata, {'type': [None, dict]})
    argspec.validate('cwd', cwd, {'type': [None, str]})
    argspec.validate('environ', environ, {'type': [None, dict]})
    argspec.validate('frameworks', frameworks, {'type': [list, tuple], 'items': {'type': dict}})

    # Make sure we have a copy of the implicit_deps so we can modify
    # it safely.
    implicit_deps = list(implicit_deps)

    # Expand Target objects listed in the commands.
    new_commands = []
    for command in commands:
      new_command = []
      for index, arg in enumerate(command):
        if isinstance(arg, Target):
          if index == 0 and len(arg.outputs) != 1:
            raise ValueError('Target as program in commands list must have '
                'exactly one output, has {} (target: {})'.format(
                  len(arg.outputs), arg.name))
          new_command += arg.outputs
          implicit_deps += arg.outputs
        else:
          new_command.append(arg)
      new_commands.append(new_command)

    self.name = name
    self.commands = new_commands
    self.inputs = list(map(path.norm, inputs))
    self.outputs = list(map(path.norm, outputs))
    self.implicit_deps = list(map(path.norm, implicit_deps))
    self.order_only_deps = list(map(path.norm, order_only_deps))
    self.pool = pool
    self.deps = deps
    self.depfile = depfile
    self.msvc_deps_prefix = msvc_deps_prefix
    self.explicit = explicit
    self.foreach = foreach
    self.description = description
    self.metadata = metadata or {}
    self.cwd = cwd
    self.environ = environ or {}
    self.frameworks = frameworks

    if self.foreach and len(self.inputs) != len(self.outputs):
      raise ValueError('foreach target must have the same number of output '
        'files ({}) as input files ({})'.format(len(self.outputs),
        len(self.inputs)))

    if self.deps == 'gcc' and not self.depfile:
      raise ValueError('require depfile with deps="gcc"')
Example #15
0
    def __init__(self,
                 name,
                 commands,
                 inputs,
                 outputs,
                 implicit_deps=(),
                 order_only_deps=(),
                 pool=None,
                 deps=None,
                 depfile=None,
                 msvc_deps_prefix=None,
                 explicit=False,
                 foreach=False,
                 description=None,
                 metadata=None,
                 cwd=None,
                 environ=None,
                 frameworks=(),
                 task=None,
                 runprefix=None):
        argspec.validate('name', name, {'type': str})
        argspec.validate(
            'commands', commands, {
                'type': list,
                'allowEmpty': False,
                'items': {
                    'type': list,
                    'allowEmpty': False,
                    'items': {
                        'type': [Tool, Target, str]
                    }
                }
            })
        argspec.validate('inputs', inputs, {
            'type': [list, tuple],
            'items': {
                'type': [Target, str]
            }
        })
        argspec.validate('outputs', outputs, {
            'type': [list, tuple],
            'items': {
                'type': str
            }
        })
        argspec.validate('implicit_deps', implicit_deps, {
            'type': [list, tuple],
            'items': {
                'type': [Target, str]
            }
        })
        argspec.validate('order_only_deps', order_only_deps, {
            'type': [list, tuple],
            'items': {
                'type': [Target, str]
            }
        })
        argspec.validate('pool', pool, {'type': [None, str]})
        argspec.validate('deps', deps, {
            'type': [None, str],
            'enum': ['msvc', 'gcc']
        })
        argspec.validate('depfile', depfile, {'type': [None, str]})
        argspec.validate('msvc_deps_prefix', msvc_deps_prefix,
                         {'type': [None, str]})
        argspec.validate('explicit', explicit, {'type': bool})
        argspec.validate('foreach', foreach, {'type': bool})
        argspec.validate('description', description, {'type': [None, str]})
        argspec.validate('metadata', metadata, {'type': [None, dict]})
        argspec.validate('cwd', cwd, {'type': [None, str]})
        argspec.validate('environ', environ, {'type': [None, dict]})
        argspec.validate('frameworks', frameworks, {
            'type': [list, tuple],
            'items': {
                'type': dict
            }
        })
        argspec.validate('task', task, {'type': [None, Task]})
        argspec.validate('runprefix', runprefix, {
            'type': [None, list, str],
            'items': {
                'type': str
            }
        })

        if isinstance(runprefix, str):
            runprefix = shell.split(runprefix)
        elif runprefix is None:
            runprefix = []

        def expand_mixed_list(mixed, implicit_deps, mode):
            result = []
            for item in mixed:
                if isinstance(item, Target):
                    if mode == 'implicit':
                        names = [item.name]
                    elif mode == 'inputs':
                        names = item.outputs
                    elif mode == 'cmd':
                        names = [path.abs(x) for x in item.outputs]
                    else:
                        raise RuntimeError(mode)

                    if implicit_deps is not None:
                        implicit_deps += names
                    result += names

                elif isinstance(item, Tool):
                    result.append(str(item))

                elif isinstance(item, str):
                    if mode != 'cmd':
                        item = path.abs(item)
                    result.append(item)

                else:
                    raise RuntimeError

            return result

        self.implicit_deps = []
        self.inputs = expand_mixed_list(inputs, None, 'inputs')
        self.outputs = [path.abs(x) for x in outputs]
        self.commands = [
            expand_mixed_list(cmd, self.implicit_deps, 'cmd')
            for cmd in commands
        ]
        self.implicit_deps += expand_mixed_list(implicit_deps, None,
                                                'implicit')
        self.order_only_deps = expand_mixed_list(order_only_deps, None,
                                                 'implicit')

        self.name = name
        self.pool = pool
        self.deps = deps
        self.depfile = depfile
        self.msvc_deps_prefix = msvc_deps_prefix
        self.explicit = explicit
        self.foreach = foreach
        self.description = description
        self.metadata = metadata or {}
        self.cwd = cwd
        self.environ = environ or {}
        self.frameworks = frameworks
        self.task = task
        self.runprefix = runprefix

        if self.foreach and len(self.inputs) != len(self.outputs):
            raise ValueError(
                'foreach target must have the same number of output '
                'files ({}) as input files ({})'.format(
                    len(self.outputs), len(self.inputs)))

        if self.deps == 'gcc' and not self.depfile:
            raise ValueError('require depfile with deps="gcc"')
Example #16
0
def download_file(url, filename=None, file=None, directory=None,
    on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None):
  """
  Download a file from a URL to one of the following destinations:

  :param filename: A filename to write the downloaded file to.
  :param file: A file-like object.
  :param directory: A directory. The filename will be automatically
    determined from the ``Content-Disposition`` header received from
    the server or the last path elemnet in the URL.

  Additional parameters for the *directory* parameter:

  :param on_exists: The operation to perform when the file already exists.
    Available modes are ``rename``, ``overwrite`` and ``skip``.

  Additional parameters:

  :param progress: A callable that accepts a single parameter that is a
    dictionary with information about the progress of the download. The
    dictionary provides the keys ``size``,  ``downloaded`` and ``response``.
    If the callable returns :const:`False` (specifically the value False), the
    download will be aborted and a :class:`UserInterrupt` will be raised.
  :param urlopen_kwargs: A dictionary with additional keyword arguments
    for :func:`urllib.request.urlopen`.

  Raise and return:

  :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
  :raise URLError: Can be raised by :func:`urllib.request.urlopen`.
  :raise UserInterrupt: If the *progress* returned :const:`False`.
  :return: If the download mode is *directory*, the name of the downloaded
    file will be returned and False if the file was newly downloaded, True
    if the download was skipped because the file already existed.
    Otherwise, the number of bytes downloaded will be returned.
  """

  argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']})

  if sum(map(bool, [filename, file, directory])) != 1:
    raise ValueError('exactly one of filename, file or directory must be specifed')

  response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))

  if directory:
    try:
      filename = parse_content_disposition(
        response.headers.get('Content-Disposition', ''))
    except ValueError:
      filename = url.split('/')[-1]
    filename = path.join(directory, filename)
    path.makedirs(directory)

    if path.exists(filename):
      if on_exists == 'skip':
        return filename, True
      elif on_exists == 'rename':
        index = 0
        while True:
          new_filename = filename + '_{:0>4}'.format(index)
          if not path.exists(new_filename):
            filename = new_filename
            break
          index += 1
      elif on_exists != 'overwrite':
        raise RuntimeError

  try:
    size = int(response.headers.get('Content-Length', ''))
  except ValueError:
    size = None

  progress_info = {'response': response, 'size': size, 'downloaded': 0,
    'completed': False, 'filename': filename, 'url': url}
  if progress and progress(progress_info) is False:
    raise UserInterrupt

  def copy_to_file(fp):
    while True:
      data = response.read(chunksize)
      if not data:
        break
      progress_info['downloaded'] += len(data)
      fp.write(data)
      if progress and progress(progress_info) is False:
        raise UserInterrupt
    progress_info['completed'] = True
    if progress and progress(progress_info) is False:
      raise UserInterrupt

  if filename:
    path.makedirs(path.dirname(filename))
    try:
      with open(filename, 'wb') as fp:
        copy_to_file(fp)
    except BaseException:
      # Delete the file if it could not be downloaded successfully.
      path.remove(filename, silent=True)
      raise
  elif file:
    copy_to_file(file)

  return filename, False
Example #17
0
def glob(patterns,
         parent=None,
         excludes=(),
         include_dotfiles=False,
         ignore_false_excludes=False):
    """
  Wrapper for :func:`glob2.glob` that accepts an arbitrary number of
  patterns and matches them. The paths are normalized with :func:`norm`.

  Relative patterns are automaticlly joined with *parent*. If the
  parameter is omitted, it defaults to the currently executed build
  scripts project directory.

  If *excludes* is specified, it must be a string or a list of strings
  that is/contains glob patterns or filenames to be removed from the
  result before returning.

  .. note::

    Every file listed in *excludes* will only remove **one** item from
    the result list that was generated from *patterns*. Thus, if you
    want to exclude some files with a pattern except for a specific file
    that would also match that pattern, simply list that file another
    time in the *patterns*.

  :param patterns: A list of glob patterns or filenames.
  :param parent: The parent directory for relative paths.
  :param excludes: A list of glob patterns or filenames.
  :param include_dotfiles: If True, ``*`` and ``**`` can also capture
    file or directory names starting with a dot.
  :param ignore_false_excludes: False by default. If True, items listed
    in *excludes* that have not been globbed will raise an exception.
  :return: A list of filenames.
  """

    argspec.validate('patterns', patterns, {'type': [list, tuple, str]})
    argspec.validate('excludes', excludes, {'type': [list, tuple]})
    argspec.validate('parent', parent, {'type': [None, str]})

    if isinstance(patterns, str):
        patterns = [patterns]

    if not parent:
        parent = getcwd()

    result = []
    for pattern in patterns:
        if not isabs(pattern):
            pattern = join(parent, pattern)
        result += glob2.glob(norm(pattern))

    for pattern in excludes:
        if not isabs(pattern):
            pattern = join(parent, pattern)
        pattern = norm(pattern)
        if not isglob(pattern):
            try:
                result.remove(pattern)
            except ValueError as exc:
                if not ignore_false_excludes:
                    raise ValueError('{} ({})'.format(exc, pattern))
        else:
            for item in glob2.glob(pattern):
                try:
                    result.remove(item)
                except ValueError as exc:
                    if not ignore_false_excludes:
                        raise ValueError('{} ({})'.format(exc, pattern))

    return result
Example #18
0
  def __init__(self, name, option_kwargs=None, frameworks=(), inputs=(),
      outputs=(), implicit_deps=(), order_only_deps=()):
    argspec.validate('name', name, {'type': str})
    argspec.validate('option_kwargs', option_kwargs,
        {'type': [None, dict, Framework]})
    argspec.validate('frameworks', frameworks,
        {'type': [list, tuple], 'items': {'type': Framework}})
    argspec.validate('inputs', inputs,
        {'type': [list, tuple, build.Target],
          'items': {'type': [str, build.Target]}})
    argspec.validate('outputs', outputs,
        {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('implicit_deps', implicit_deps,
        {'type': [list, tuple], 'items': {'type': str}})
    argspec.validate('order_only_deps', order_only_deps,
        {'type': [list, tuple], 'items': {'type': str}})

    self.frameworks = list(frameworks)
    if isinstance(inputs, build.Target):
      inputs = [inputs]

    # If we find any Target objects in the inputs, expand the outputs
    # and append the frameworks.
    if inputs is not None:
      self.inputs = []
      for input_ in (inputs or ()):
        if isinstance(input_, build.Target):
          pyutils.unique_extend(self.frameworks, input_.frameworks)
          self.inputs += input_.outputs
        else:
          self.inputs.append(input_)
    else:
      self.inputs = None

    if option_kwargs is None:
      option_kwargs = {}

    self.name = name
    self.option_kwargs = Framework(name, **option_kwargs)
    self.option_kwargs_defaults = Framework(name + "_defaults")
    self.options_merge = OptionMerge(self.option_kwargs,
        self.option_kwargs_defaults, *self.frameworks)
    assert self.option_kwargs in self.options_merge.frameworks
    self.outputs = list(outputs)
    self.implicit_deps = list(implicit_deps)
    self.order_only_deps = list(order_only_deps)
    self.metadata = {}
    self.used_option_keys = set()