def find_module(self, name, version): """ Finds a module in the :attr:`path` matching the specified *name* and *version*. :param name: The name of the module. :param version: A :class:`VersionCriteria`, :class:`Version` or string in a VersionCritiera format. :raise ModuleNotFound: If the module can not be found. :return: :class:`Module` """ argspec.validate('name', name, {'type': str}) argspec.validate('version', version, {'type': [str, Version, VersionCriteria]}) if isinstance(version, str): try: version = Version(version) except ValueError as exc: version = VersionCriteria(version) self.update_manifest_cache() if name in self.modules: if isinstance(version, Version): if version in self.modules[name]: return self.modules[name][version] raise ModuleNotFound(name, version) for module in sorted(self.modules[name].values(), key=lambda x: x.manifest.version, reverse=True): if version(module.manifest.version): return module raise ModuleNotFound(name, version)
def export(self, writer, context, platform): """ Export the build graph to a Ninja manifest. :param writer: A :class:`ninja_syntax.Writer` object. :param context: A :class:`ExportContext` object. :param platform: A :class:`PlatformHelper` instance. """ argspec.validate('writer', writer, {"type": ninja_syntax.Writer}) writer.comment('This file was automatically generated with Craftr.') writer.comment('It is not recommended to edit this file manually.') writer.newline() if self.vars: for key, value in self.vars.items(): writer.variable(key, value) writer.newline() if self.tools: writer.comment('Tools') writer.comment('-----') for tool in self.tools.values(): tool.export(writer, context, platform) writer.newline() defaults = [] for target in self.targets.values(): if not target.explicit: defaults.append(target.name) target.export(writer, context, platform) if defaults: writer.default(defaults)
def export(self, writer, context, platform): """ Export the build graph to a Ninja manifest. :param writer: A :class:`ninja_syntax.Writer` object. :param context: A :class:`ExportContext` object. :param platform: A :class:`PlatformHelper` instance. """ argspec.validate('writer', writer, {"type": ninja_syntax.Writer}) writer.comment('This file was automatically generated with Craftr.') writer.comment('It is not recommended to edit this file manually.') writer.newline() if self.vars: for key, value in self.vars.items(): writer.variable(key, value) writer.newline() if self.tools: writer.comment('Tools') writer.comment('-----') for tool in self.tools.values(): tool.export(writer, context, platform) writer.newline() defaults = [] for target in self.targets.values(): if not target.explicit and target.generates_build_instruction: defaults.append(target.name) target.export(writer, context, platform) if defaults: writer.default(defaults)
def find_module(self, name, version, resolve_preferred_version=True): """ Finds a module in the :attr:`path` matching the specified *name* and *version*. :param name: The name of the module. :param version: A :class:`VersionCriteria`, :class:`Version` or string in a VersionCritiera format. :param resolve_preferred_version: If this parameter is True (default) and a preferred version is specified in :attr:`preferred_versions`, that preferred version is loaded or :class:`ModuleNotFound` is raised. :raise ModuleNotFound: If the module can not be found. :return: :class:`Module` """ argspec.validate('name', name, {'type': str}) argspec.validate('version', version, {'type': [str, Version, VersionCriteria]}) if name in renames.renames: logger.warn('"{}" is deprecated, use "{}" instead'.format( name, renames.renames[name])) name = renames.renames[name] if isinstance(version, str): try: version = Version(version) except ValueError as exc: version = VersionCriteria(version) if session.module and resolve_preferred_version: data = self.preferred_versions.get(session.module.manifest.name) if data is not None: versions = data.get(str(session.module.manifest.version)) if versions is not None: preferred_version = versions.get(name) if preferred_version is not None: version = Version(preferred_version) logger.debug( 'note: loading preferred version {} of module "{}" ' 'requested by module "{}"'.format( version, name, session.module.ident)) self.update_manifest_cache() if name in self.modules: if isinstance(version, Version): if version in self.modules[name]: return self.modules[name][version] raise ModuleNotFound(name, version) for module in sorted(self.modules[name].values(), key=lambda x: x.manifest.version, reverse=True): if version(module.manifest.version): return module raise ModuleNotFound(name, version)
def __init__(self, name, command, preamble=None, environ=None): argspec.validate('name', name, {'type': str}) argspec.validate('command', command, { 'type': [list, tuple], 'allowEmpty': False, 'items': { 'type': str } }) argspec.validate( 'preamble', preamble, { 'type': [None, list, tuple], 'items': { 'type': [list, tuple], 'allowEmpty': False, 'items': { 'type': str } } }) argspec.validate('environ', environ, {'type': [None, dict]}) self.name = name self.command = list(command) self.preamble = preamble or [] self.environ = environ or {} self.exported_command = None
def add_task(self, task, **kwargs): """ Add a :class:`Task` to the Graph. Creates a :class:`Target` for the task and returns it. :param task: The :class:`Task` to add to the Graph. :param kwargs: Additional parameters for the :class:`Target` constructor. """ argspec.validate('task', task, {'type': Task}) if task.name in self.tasks: raise ValueError('a task with the name {!r} already exists'.format( task.name)) target = Target(task.name, [task.get_command()], task=task, **kwargs) assert target.task is task self.add_target(target) self.tasks[task.name] = task return target
def __init__(self, name, command, preamble=None, environ=None): argspec.validate('name', name, {'type': str}) argspec.validate('command', command, {'type': [list, tuple], 'allowEmpty': False, 'items': {'type': str}}) argspec.validate('preamble', preamble, {'type': [None, list, tuple], 'items': {'type': [list, tuple], 'allowEmpty': False, 'items': {'type': str}}}) argspec.validate('environ', environ, {'type': [None, dict]}) self.name = name self.command = command self.preamble = preamble or [] self.environ = environ or {} self.exported_command = None
def add_target(self, target): """ Add a :class:`Target` to the Graph. .. note:: For performance reasons, this method assumes that all paths in the *target* are already normalized with :meth:`path.norm`. :raise ValueError: If the :attr:`Target.name` is already used. :raise DuplicateOutputError: If the *target* lists an output file that is already created by another target. """ argspec.validate('target', target, {'type': Target}) if target.name in self.targets: raise ValueError('a target with the name {!r} already exists' .format(target.name)) self.targets[target.name] = target for infile in target.inputs: self.infiles.setdefault(infile, []).append(target) for outfile in target.outputs: other = self.outfiles.setdefault(outfile, target) if other is not target: raise DuplicateOutputError(outfile, target, other)
def add_target(self, target): """ Add a :class:`Target` to the Graph. .. note:: For performance reasons, this method assumes that all paths in the *target* are already normalized with :meth:`path.norm`. :raise ValueError: If the :attr:`Target.name` is already used. :raise DuplicateOutputError: If the *target* lists an output file that is already created by another target. """ argspec.validate('target', target, {'type': Target}) if target.name in self.targets: raise ValueError( 'a target with the name {!r} already exists'.format( target.name)) self.targets[target.name] = target for infile in target.inputs: self.infiles.setdefault(infile, []).append(target) for outfile in target.outputs: other = self.outfiles.setdefault(outfile, target) if other is not target: raise DuplicateOutputError(outfile, target, other)
def glob(patterns, parent=None, excludes=(), include_dotfiles=False): """ Wrapper for :func:`glob2.glob` that accepts an arbitrary number of patterns and matches them. The paths are normalized with :func:`norm`. Relative patterns are automaticlly joined with *parent*. If the parameter is omitted, it defaults to the currently executed build scripts project directory. If *excludes* is specified, it must be a string or a list of strings that is/contains glob patterns or filenames to be removed from the result before returning. .. note:: Every file listed in *excludes* will only remove **one** item from the result list that was generated from *patterns*. Thus, if you want to exclude some files with a pattern except for a specific file that would also match that pattern, simply list that file another time in the *patterns*. :param patterns: A list of glob patterns or filenames. :param parent: The parent directory for relative paths. :param excludes: A list of glob patterns or filenames. :param include_dotfiles: If True, ``*`` and ``**`` can also capture file or directory names starting with a dot. :return: A list of filenames. """ argspec.validate("patterns", patterns, {"type": [list, tuple]}) argspec.validate("excludes", excludes, {"type": [list, tuple]}) argspec.validate("parent", parent, {"type": [None, str]}) if not parent: parent = getcwd() result = [] for pattern in patterns: if not isabs(pattern): pattern = join(parent, pattern) result += glob2.glob(norm(pattern)) for pattern in excludes: if not isabs(pattern): pattern = join(parent, pattern) pattern = norm(pattern) if not isglob(pattern): result.remove(pattern) else: for item in glob2.glob(pattern): result.remove(item) return result
def __init__(self, name, commands, inputs, outputs, implicit_deps=(), order_only_deps=(), pool=None, deps=None, depfile=None, msvc_deps_prefix=None, explicit=False, foreach=False, description=None, metadata=None, cwd=None, environ=None, frameworks=()): argspec.validate('name', name, {'type': str}) argspec.validate('commands', commands, {'type': list, 'allowEmpty': False, 'items': {'type': list, 'allowEmpty': False, 'items': {'type': [Tool, Target, str]}}}) argspec.validate('inputs', inputs, {'type': [list, tuple], 'items': {'type': [Target, str]}}) argspec.validate('outputs', outputs, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('implicit_deps', implicit_deps, {'type': [list, tuple], 'items': {'type': [Target, str]}}) argspec.validate('order_only_deps', order_only_deps, {'type': [list, tuple], 'items': {'type': [Target, str]}}) argspec.validate('pool', pool, {'type': [None, str]}) argspec.validate('deps', deps, {'type': [None, str], 'enum': ['msvc', 'gcc']}) argspec.validate('depfile', depfile, {'type': [None, str]}) argspec.validate('msvc_deps_prefix', msvc_deps_prefix, {'type': [None, str]}) argspec.validate('explicit', explicit, {'type': bool}) argspec.validate('foreach', foreach, {'type': bool}) argspec.validate('description', description, {'type': [None, str]}) argspec.validate('metadata', metadata, {'type': [None, dict]}) argspec.validate('cwd', cwd, {'type': [None, str]}) argspec.validate('environ', environ, {'type': [None, dict]}) argspec.validate('frameworks', frameworks, {'type': [list, tuple], 'items': {'type': dict}}) def expand_mixed_list(mixed, implicit_deps, mode): result = [] for item in mixed: if isinstance(item, Target): if mode == 'implicit': names = [item.name] elif mode == 'inputs': names = item.outputs elif mode == 'cmd': names = [path.abs(x) for x in item.outputs] else: raise RuntimeError(mode) if implicit_deps is not None: implicit_deps += names result += names elif isinstance(item, str): if mode != 'cmd': item = path.abs(item) result.append(item) else: raise RuntimeError return result self.implicit_deps = [] self.inputs = expand_mixed_list(inputs, None, 'inputs') self.outputs = [path.abs(x) for x in outputs] self.commands = [expand_mixed_list(cmd, self.implicit_deps, 'cmd') for cmd in commands] self.implicit_deps += expand_mixed_list(implicit_deps, None, 'implicit') self.order_only_deps = expand_mixed_list(order_only_deps, None, 'implicit') self.name = name self.pool = pool self.deps = deps self.depfile = depfile self.msvc_deps_prefix = msvc_deps_prefix self.explicit = explicit self.foreach = foreach self.description = description self.metadata = metadata or {} self.cwd = cwd self.environ = environ or {} self.frameworks = frameworks if self.foreach and len(self.inputs) != len(self.outputs): raise ValueError('foreach target must have the same number of output ' 'files ({}) as input files ({})'.format(len(self.outputs), len(self.inputs))) if self.deps == 'gcc' and not self.depfile: raise ValueError('require depfile with deps="gcc"')
def download_file(url, filename=None, file=None, directory=None, on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None): """ Download a file from a URL to one of the following destinations: :param filename: A filename to write the downloaded file to. :param file: A file-like object. :param directory: A directory. The filename will be automatically determined from the ``Content-Disposition`` header received from the server or the last path elemnet in the URL. Additional parameters for the *directory* parameter: :param on_exists: The operation to perform when the file already exists. Available modes are ``rename``, ``overwrite`` and ``skip``. Additional parameters: :param progress: A callable that accepts a single parameter that is a dictionary with information about the progress of the download. The dictionary provides the keys ``size``, ``downloaded`` and ``response``. If the callable returns :const:`False` (specifically the value False), the download will be aborted and a :class:`UserInterrupt` will be raised. :param urlopen_kwargs: A dictionary with additional keyword arguments for :func:`urllib.request.urlopen`. Raise and return: :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`. :raise URLError: Can be raised by :func:`urllib.request.urlopen`. :raise UserInterrupt: If the *progress* returned :const:`False`. :return: If the download mode is *directory*, the name of the downloaded file will be returned and False if the file was newly downloaded, True if the download was skipped because the file already existed. Otherwise, the number of bytes downloaded will be returned. """ argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']}) if sum(map(bool, [filename, file, directory])) != 1: raise ValueError( 'exactly one of filename, file or directory must be specifed') response = urllib.request.urlopen(url, **(urlopen_kwargs or {})) if directory: try: filename = parse_content_disposition( response.headers.get('Content-Disposition', '')) except ValueError: filename = url.split('/')[-1] filename = path.join(directory, filename) path.makedirs(directory) if path.exists(filename): if on_exists == 'skip': return filename, True elif on_exists == 'rename': index = 0 while True: new_filename = filename + '_{:0>4}'.format(index) if not path.exists(new_filename): filename = new_filename break index += 1 elif on_exists != 'overwrite': raise RuntimeError try: size = int(response.headers.get('Content-Length', '')) except ValueError: size = None progress_info = { 'response': response, 'size': size, 'downloaded': 0, 'completed': False, 'filename': filename, 'url': url } if progress and progress(progress_info) is False: raise UserInterrupt def copy_to_file(fp): while True: data = response.read(chunksize) if not data: break progress_info['downloaded'] += len(data) fp.write(data) if progress and progress(progress_info) is False: raise UserInterrupt progress_info['completed'] = True if progress and progress(progress_info) is False: raise UserInterrupt if filename: path.makedirs(path.dirname(filename)) try: with open(filename, 'wb') as fp: copy_to_file(fp) except BaseException: # Delete the file if it could not be downloaded successfully. path.remove(filename, silent=True) raise elif file: copy_to_file(file) return filename, False
def __init__(self, name, option_kwargs=None, frameworks=(), inputs=(), outputs=(), implicit_deps=(), order_only_deps=()): argspec.validate('name', name, {'type': str}) argspec.validate('option_kwargs', option_kwargs, {'type': [None, dict, Framework]}) argspec.validate('frameworks', frameworks, { 'type': [list, tuple], 'items': { 'type': [Framework, build.Target] } }) argspec.validate( 'inputs', inputs, { 'type': [list, tuple, build.Target], 'items': { 'type': [str, build.Target] } }) argspec.validate('outputs', outputs, { 'type': [list, tuple], 'items': { 'type': str } }) argspec.validate('implicit_deps', implicit_deps, { 'type': [list, tuple], 'items': { 'type': str } }) argspec.validate('order_only_deps', order_only_deps, { 'type': [list, tuple], 'items': { 'type': str } }) if isinstance(inputs, build.Target): inputs = [inputs] self.frameworks = [] self.implicit_deps = list(implicit_deps) self.order_only_deps = list(order_only_deps) # If we find any Target objects in the inputs, expand the outputs # and append the frameworks. if inputs is not None: self.inputs = [] for input_ in (inputs or ()): if isinstance(input_, build.Target): pyutils.unique_extend(self.frameworks, input_.frameworks) self.inputs += input_.outputs else: self.inputs.append(input_) else: self.inputs = None # Unpack the frameworks list, which may also container Targets. for fw in frameworks: if isinstance(fw, build.Target): self.implicit_deps.append(fw) self.frameworks += fw.frameworks else: self.frameworks.append(fw) if option_kwargs is None: option_kwargs = {} self.name = name self.outputs = list(outputs) self.metadata = {} self.used_option_keys = set() self.option_kwargs = Framework(name, **option_kwargs) self.option_kwargs_defaults = Framework(name + "_defaults") self.options_merge = OptionMerge(self.option_kwargs, self.option_kwargs_defaults, *self.frameworks) assert self.option_kwargs in self.options_merge.frameworks
def __init__(self, name, commands, inputs, outputs, implicit_deps=(), order_only_deps=(), pool=None, deps=None, depfile=None, msvc_deps_prefix=None, explicit=False, foreach=False, description=None, metadata=None, cwd=None, environ=None, frameworks=()): argspec.validate('name', name, {'type': str}) argspec.validate('commands', commands, {'type': list, 'allowEmpty': False, 'items': {'type': list, 'allowEmpty': False, 'items': {'type': [Tool, Target, str]}}}) argspec.validate('inputs', inputs, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('outputs', outputs, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('implicit_deps', implicit_deps, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('order_only_deps', order_only_deps, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('pool', pool, {'type': [None, str]}) argspec.validate('deps', deps, {'type': [None, str], 'enum': ['msvc', 'gcc']}) argspec.validate('depfile', depfile, {'type': [None, str]}) argspec.validate('msvc_deps_prefix', msvc_deps_prefix, {'type': [None, str]}) argspec.validate('explicit', explicit, {'type': bool}) argspec.validate('foreach', foreach, {'type': bool}) argspec.validate('description', description, {'type': [None, str]}) argspec.validate('metadata', metadata, {'type': [None, dict]}) argspec.validate('cwd', cwd, {'type': [None, str]}) argspec.validate('environ', environ, {'type': [None, dict]}) argspec.validate('frameworks', frameworks, {'type': [list, tuple], 'items': {'type': dict}}) # Make sure we have a copy of the implicit_deps so we can modify # it safely. implicit_deps = list(implicit_deps) # Expand Target objects listed in the commands. new_commands = [] for command in commands: new_command = [] for index, arg in enumerate(command): if isinstance(arg, Target): if index == 0 and len(arg.outputs) != 1: raise ValueError('Target as program in commands list must have ' 'exactly one output, has {} (target: {})'.format( len(arg.outputs), arg.name)) new_command += arg.outputs implicit_deps += arg.outputs else: new_command.append(arg) new_commands.append(new_command) self.name = name self.commands = new_commands self.inputs = list(map(path.norm, inputs)) self.outputs = list(map(path.norm, outputs)) self.implicit_deps = list(map(path.norm, implicit_deps)) self.order_only_deps = list(map(path.norm, order_only_deps)) self.pool = pool self.deps = deps self.depfile = depfile self.msvc_deps_prefix = msvc_deps_prefix self.explicit = explicit self.foreach = foreach self.description = description self.metadata = metadata or {} self.cwd = cwd self.environ = environ or {} self.frameworks = frameworks if self.foreach and len(self.inputs) != len(self.outputs): raise ValueError('foreach target must have the same number of output ' 'files ({}) as input files ({})'.format(len(self.outputs), len(self.inputs))) if self.deps == 'gcc' and not self.depfile: raise ValueError('require depfile with deps="gcc"')
def __init__(self, name, commands, inputs, outputs, implicit_deps=(), order_only_deps=(), pool=None, deps=None, depfile=None, msvc_deps_prefix=None, explicit=False, foreach=False, description=None, metadata=None, cwd=None, environ=None, frameworks=(), task=None, runprefix=None): argspec.validate('name', name, {'type': str}) argspec.validate( 'commands', commands, { 'type': list, 'allowEmpty': False, 'items': { 'type': list, 'allowEmpty': False, 'items': { 'type': [Tool, Target, str] } } }) argspec.validate('inputs', inputs, { 'type': [list, tuple], 'items': { 'type': [Target, str] } }) argspec.validate('outputs', outputs, { 'type': [list, tuple], 'items': { 'type': str } }) argspec.validate('implicit_deps', implicit_deps, { 'type': [list, tuple], 'items': { 'type': [Target, str] } }) argspec.validate('order_only_deps', order_only_deps, { 'type': [list, tuple], 'items': { 'type': [Target, str] } }) argspec.validate('pool', pool, {'type': [None, str]}) argspec.validate('deps', deps, { 'type': [None, str], 'enum': ['msvc', 'gcc'] }) argspec.validate('depfile', depfile, {'type': [None, str]}) argspec.validate('msvc_deps_prefix', msvc_deps_prefix, {'type': [None, str]}) argspec.validate('explicit', explicit, {'type': bool}) argspec.validate('foreach', foreach, {'type': bool}) argspec.validate('description', description, {'type': [None, str]}) argspec.validate('metadata', metadata, {'type': [None, dict]}) argspec.validate('cwd', cwd, {'type': [None, str]}) argspec.validate('environ', environ, {'type': [None, dict]}) argspec.validate('frameworks', frameworks, { 'type': [list, tuple], 'items': { 'type': dict } }) argspec.validate('task', task, {'type': [None, Task]}) argspec.validate('runprefix', runprefix, { 'type': [None, list, str], 'items': { 'type': str } }) if isinstance(runprefix, str): runprefix = shell.split(runprefix) elif runprefix is None: runprefix = [] def expand_mixed_list(mixed, implicit_deps, mode): result = [] for item in mixed: if isinstance(item, Target): if mode == 'implicit': names = [item.name] elif mode == 'inputs': names = item.outputs elif mode == 'cmd': names = [path.abs(x) for x in item.outputs] else: raise RuntimeError(mode) if implicit_deps is not None: implicit_deps += names result += names elif isinstance(item, Tool): result.append(str(item)) elif isinstance(item, str): if mode != 'cmd': item = path.abs(item) result.append(item) else: raise RuntimeError return result self.implicit_deps = [] self.inputs = expand_mixed_list(inputs, None, 'inputs') self.outputs = [path.abs(x) for x in outputs] self.commands = [ expand_mixed_list(cmd, self.implicit_deps, 'cmd') for cmd in commands ] self.implicit_deps += expand_mixed_list(implicit_deps, None, 'implicit') self.order_only_deps = expand_mixed_list(order_only_deps, None, 'implicit') self.name = name self.pool = pool self.deps = deps self.depfile = depfile self.msvc_deps_prefix = msvc_deps_prefix self.explicit = explicit self.foreach = foreach self.description = description self.metadata = metadata or {} self.cwd = cwd self.environ = environ or {} self.frameworks = frameworks self.task = task self.runprefix = runprefix if self.foreach and len(self.inputs) != len(self.outputs): raise ValueError( 'foreach target must have the same number of output ' 'files ({}) as input files ({})'.format( len(self.outputs), len(self.inputs))) if self.deps == 'gcc' and not self.depfile: raise ValueError('require depfile with deps="gcc"')
def download_file(url, filename=None, file=None, directory=None, on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None): """ Download a file from a URL to one of the following destinations: :param filename: A filename to write the downloaded file to. :param file: A file-like object. :param directory: A directory. The filename will be automatically determined from the ``Content-Disposition`` header received from the server or the last path elemnet in the URL. Additional parameters for the *directory* parameter: :param on_exists: The operation to perform when the file already exists. Available modes are ``rename``, ``overwrite`` and ``skip``. Additional parameters: :param progress: A callable that accepts a single parameter that is a dictionary with information about the progress of the download. The dictionary provides the keys ``size``, ``downloaded`` and ``response``. If the callable returns :const:`False` (specifically the value False), the download will be aborted and a :class:`UserInterrupt` will be raised. :param urlopen_kwargs: A dictionary with additional keyword arguments for :func:`urllib.request.urlopen`. Raise and return: :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`. :raise URLError: Can be raised by :func:`urllib.request.urlopen`. :raise UserInterrupt: If the *progress* returned :const:`False`. :return: If the download mode is *directory*, the name of the downloaded file will be returned and False if the file was newly downloaded, True if the download was skipped because the file already existed. Otherwise, the number of bytes downloaded will be returned. """ argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']}) if sum(map(bool, [filename, file, directory])) != 1: raise ValueError('exactly one of filename, file or directory must be specifed') response = urllib.request.urlopen(url, **(urlopen_kwargs or {})) if directory: try: filename = parse_content_disposition( response.headers.get('Content-Disposition', '')) except ValueError: filename = url.split('/')[-1] filename = path.join(directory, filename) path.makedirs(directory) if path.exists(filename): if on_exists == 'skip': return filename, True elif on_exists == 'rename': index = 0 while True: new_filename = filename + '_{:0>4}'.format(index) if not path.exists(new_filename): filename = new_filename break index += 1 elif on_exists != 'overwrite': raise RuntimeError try: size = int(response.headers.get('Content-Length', '')) except ValueError: size = None progress_info = {'response': response, 'size': size, 'downloaded': 0, 'completed': False, 'filename': filename, 'url': url} if progress and progress(progress_info) is False: raise UserInterrupt def copy_to_file(fp): while True: data = response.read(chunksize) if not data: break progress_info['downloaded'] += len(data) fp.write(data) if progress and progress(progress_info) is False: raise UserInterrupt progress_info['completed'] = True if progress and progress(progress_info) is False: raise UserInterrupt if filename: path.makedirs(path.dirname(filename)) try: with open(filename, 'wb') as fp: copy_to_file(fp) except BaseException: # Delete the file if it could not be downloaded successfully. path.remove(filename, silent=True) raise elif file: copy_to_file(file) return filename, False
def glob(patterns, parent=None, excludes=(), include_dotfiles=False, ignore_false_excludes=False): """ Wrapper for :func:`glob2.glob` that accepts an arbitrary number of patterns and matches them. The paths are normalized with :func:`norm`. Relative patterns are automaticlly joined with *parent*. If the parameter is omitted, it defaults to the currently executed build scripts project directory. If *excludes* is specified, it must be a string or a list of strings that is/contains glob patterns or filenames to be removed from the result before returning. .. note:: Every file listed in *excludes* will only remove **one** item from the result list that was generated from *patterns*. Thus, if you want to exclude some files with a pattern except for a specific file that would also match that pattern, simply list that file another time in the *patterns*. :param patterns: A list of glob patterns or filenames. :param parent: The parent directory for relative paths. :param excludes: A list of glob patterns or filenames. :param include_dotfiles: If True, ``*`` and ``**`` can also capture file or directory names starting with a dot. :param ignore_false_excludes: False by default. If True, items listed in *excludes* that have not been globbed will raise an exception. :return: A list of filenames. """ argspec.validate('patterns', patterns, {'type': [list, tuple, str]}) argspec.validate('excludes', excludes, {'type': [list, tuple]}) argspec.validate('parent', parent, {'type': [None, str]}) if isinstance(patterns, str): patterns = [patterns] if not parent: parent = getcwd() result = [] for pattern in patterns: if not isabs(pattern): pattern = join(parent, pattern) result += glob2.glob(norm(pattern)) for pattern in excludes: if not isabs(pattern): pattern = join(parent, pattern) pattern = norm(pattern) if not isglob(pattern): try: result.remove(pattern) except ValueError as exc: if not ignore_false_excludes: raise ValueError('{} ({})'.format(exc, pattern)) else: for item in glob2.glob(pattern): try: result.remove(item) except ValueError as exc: if not ignore_false_excludes: raise ValueError('{} ({})'.format(exc, pattern)) return result
def __init__(self, name, option_kwargs=None, frameworks=(), inputs=(), outputs=(), implicit_deps=(), order_only_deps=()): argspec.validate('name', name, {'type': str}) argspec.validate('option_kwargs', option_kwargs, {'type': [None, dict, Framework]}) argspec.validate('frameworks', frameworks, {'type': [list, tuple], 'items': {'type': Framework}}) argspec.validate('inputs', inputs, {'type': [list, tuple, build.Target], 'items': {'type': [str, build.Target]}}) argspec.validate('outputs', outputs, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('implicit_deps', implicit_deps, {'type': [list, tuple], 'items': {'type': str}}) argspec.validate('order_only_deps', order_only_deps, {'type': [list, tuple], 'items': {'type': str}}) self.frameworks = list(frameworks) if isinstance(inputs, build.Target): inputs = [inputs] # If we find any Target objects in the inputs, expand the outputs # and append the frameworks. if inputs is not None: self.inputs = [] for input_ in (inputs or ()): if isinstance(input_, build.Target): pyutils.unique_extend(self.frameworks, input_.frameworks) self.inputs += input_.outputs else: self.inputs.append(input_) else: self.inputs = None if option_kwargs is None: option_kwargs = {} self.name = name self.option_kwargs = Framework(name, **option_kwargs) self.option_kwargs_defaults = Framework(name + "_defaults") self.options_merge = OptionMerge(self.option_kwargs, self.option_kwargs_defaults, *self.frameworks) assert self.option_kwargs in self.options_merge.frameworks self.outputs = list(outputs) self.implicit_deps = list(implicit_deps) self.order_only_deps = list(order_only_deps) self.metadata = {} self.used_option_keys = set()