def write_command_file(self, filename, commands, inputs=None, outputs=None, cwd=None, environ=None, foreach=False, suffix='.cmd', dry=False, accept_additional_args=False): if suffix is not None: filename = path.addsuffix(filename, suffix) result = ['cmd', '/Q', '/c', filename] if foreach: result += ['$in', '$out'] inputs, outputs = ['%1'], ['%2'] commands = self.replace_commands_inout_vars(commands, inputs, outputs) if dry: return result, filename path.makedirs(path.dirname(path.abs(filename))) with open(filename, 'w') as fp: fp.write('REM This file is automatically generated with Craftr. It is \n') fp.write('REM not recommended to modify it manually.\n\n') if cwd is not None: fp.write('cd ' + shell.quote(cwd) + '\n\n') for key, value in environ.items(): fp.write('set ' + shell.quote('{}={}'.format(key, value), for_ninja=True) + '\n') fp.write('\n') for index, command in enumerate(commands): if accept_additional_args and index == len(commands)-1: command.append(shell.safe('%*')) fp.write(shell.join(command) + '\n') fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n') return result, filename
def execute(self, parser, args): directory = args.directory or args.name if path.maybedir(directory): directory = path.join(directory, args.name) if not path.exists(directory): logger.debug('creating directory "{}"'.format(directory)) path.makedirs(directory) elif not path.isdir(directory): logger.error('"{}" is not a directory'.format(directory)) return 1 if args.nested: directory = path.join(directory, 'craftr') path.makedirs(directory) mfile = path.join(directory, 'manifest.' + args.format) sfile = path.join(directory, 'Craftrfile') for fn in [mfile, sfile]: if path.isfile(fn): logger.error('"{}" already exists'.format(fn)) return 1 logger.debug('creating file "{}"'.format(mfile)) with open(mfile, 'w') as fp: if args.format == 'cson': lines = textwrap.dedent(''' name: "%s" version: "%s" project_dir: ".." author: "" url: "" dependencies: {} options: {} ''' % (args.name, args.version)).lstrip().split('\n') if not args.nested: del lines[2] elif args.format == 'json': lines = textwrap.dedent(''' { "name": "%s", "version": "%s", "project_dir": "..", "author": "", "url": "", "dependencies": {}, "options": {} }''' % (args.name, args.version)).lstrip().split('\n') if not args.nested: del lines[3] fp.write('\n'.join(lines)) logger.debug('creating file "{}"'.format(sfile)) with open(sfile, 'w') as fp: print('# {}'.format(args.name), file=fp)
def write_cache(cachefile): # Write back the cache. try: path.makedirs(path.dirname(cachefile)) with open(cachefile, 'w') as fp: session.write_cache(fp) except OSError as exc: logger.error('error writing cache file:', cachefile) logger.error(exc, indent=1) else: logger.debug('cache written:', cachefile)
def execute(self, parser, args): directory = args.directory or args.name if path.maybedir(directory): directory = path.join(directory, args.name) if not path.exists(directory): logger.debug('creating directory "{}"'.format(directory)) path.makedirs(directory) elif not path.isdir(directory): logger.error('"{}" is not a directory'.format(directory)) return 1 if args.nested: directory = path.join(directory, "craftr") path.makedirs(directory) mfile = path.join(directory, MANIFEST_FILENAME) sfile = path.join(directory, "Craftrfile") for fn in [mfile, sfile]: if path.isfile(fn): logger.error('"{}" already exists'.format(fn)) return 1 logger.debug('creating file "{}"'.format(mfile)) with open(mfile, "w") as fp: lines = ( textwrap.dedent( """ { "name": "%s", "version": "%s", "project_dir": "..", "author": "", "url": "", "dependencies": {}, "options": {} }\n""" % (args.name, args.version) ) .lstrip() .split("\n") ) if not args.nested: del lines[3] fp.write("\n".join(lines)) logger.debug('creating file "{}"'.format(sfile)) with open(sfile, "w") as fp: print("# {}".format(args.name), file=fp)
def write_command_file(self, filename, commands, inputs=None, outputs=None, cwd=None, environ=None, foreach=False, suffix='.sh', dry=False, accept_additional_args=False): if suffix is not None: filename = path.addsuffix(filename, suffix) result = [filename] if foreach: result += ['$in', '$out'] inputs, outputs = ['%1'], ['%2'] commands = self.replace_commands_inout_vars(commands, inputs, outputs) if dry: return result, filename path.makedirs(path.dirname(filename)) with open(filename, 'w') as fp: # TODO: Make sure this also works for shells other than bash. fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) + '\n') fp.write('set -e\n') if cwd: fp.write('cd ' + shell.quote(cwd) + '\n') fp.write('\n') for key, value in environ.items(): fp.write('export {}={}\n'.format(key, shell.quote(value))) fp.write('\n') for index, command in enumerate(commands): if accept_additional_args and index == len(commands) - 1: command.append(shell.safe('$*')) fp.write(shell.join(command)) fp.write('\n') os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # rwxrw-r-- return result, filename
def execute(self, parser, args): if hasattr(args, 'include_path'): session.path.extend(map(path.norm, args.include_path)) # Help-command preprocessing. Check if we're to show the help on a builtin # object, otherwise extract the module name if applicable. if self.mode == 'help': if not args.name: help('craftr') return 0 if args.name in vars(craftr.defaults): help(getattr(craftr.defaults, args.name)) return 0 # Check if we have an absolute symbol reference. if ':' in args.name: if args.module: parser.error( '-m/--module option conflicting with name argument: "{}"' .format(args.name)) args.module, args.name = args.name.split(':', 1) module = self._find_module(parser, args) session.main_module = module self.ninja_bin, self.ninja_version = get_ninja_info() # Create and switch to the build directory. session.builddir = path.abs(path.norm(args.build_dir, INIT_DIR)) path.makedirs(session.builddir) os.chdir(session.builddir) self.cachefile = path.join(session.builddir, '.craftrcache') # Prepare options, loaders and execute. if self.mode in ('export', 'run', 'help'): return self._export_run_or_help(args, module) elif self.mode == 'dump-options': return self._dump_options(args, module) elif self.mode == 'dump-deptree': return self._dump_deptree(args, module) elif self.mode in ('build', 'clean'): return self._build_or_clean(args) elif self.mode == 'lock': self._create_lockfile() else: raise RuntimeError("mode: {}".format(self.mode))
def write_command_file(self, filename, commands, inputs=None, outputs=None, cwd=None, environ=None, foreach=False, suffix='.cmd', dry=False, accept_additional_args=False): if suffix is not None: filename = path.addsuffix(filename, suffix) result = ['cmd', '/Q', '/c', filename] if foreach: result += ['$in', '$out'] inputs, outputs = ['%1'], ['%2'] commands = self.replace_commands_inout_vars(commands, inputs, outputs) if dry: return result, filename path.makedirs(path.dirname(path.abs(filename))) with open(filename, 'w') as fp: fp.write( 'REM This file is automatically generated with Craftr. It is \n' ) fp.write('REM not recommended to modify it manually.\n\n') if cwd is not None: fp.write('cd ' + shell.quote(cwd) + '\n\n') for key, value in environ.items(): fp.write( 'set ' + shell.quote('{}={}'.format(key, value), for_ninja=True) + '\n') fp.write('\n') for index, command in enumerate(commands): if accept_additional_args and index == len(commands) - 1: command.append(shell.safe('%*')) fp.write(shell.join(command) + '\n') fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n') return result, filename
def write_response_file(arguments, builder=None, name=None, force_file=False, suffix=''): """ Creates a response-file with the specified *name* in the in the ``buildfiles/`` directory and writes the *arguments* list quoted into the file. If *builder* is specified, it must be a :class:`TargetBuilder` and the response file will be added to the implicit dependencies. If *force_file* is set to True, a file will always be written. Otherwise, the function will into possible limitations of the platform and decide whether to write a response file or to return the *arguments* as is. Returns a tuple of ``(filename, arguments)``. If a response file is written, the returned *arguments* will be a list with a single string that is the filename prepended with ``@``. The *filename* part can be None if no response file needed to be exported. """ if not name: if not builder: raise ValueError('builder must be specified if name is bot') name = builder.name + suffix + '.response.txt' if platform.name != 'win': return None, arguments # We'll just assume that there won't be more than 2048 characters for # other flags. The windows max buffer size is 8192. content = shell.join(arguments) if len(content) < 6144: return None, arguments filename = buildlocal(path.join('buildfiles', name)) if builder: builder.implicit_deps.append(filename) if session.builddir: path.makedirs(path.dirname(filename)) with open(filename, 'w') as fp: fp.write(content) return filename, ['@' + filename]
def write_response_file(arguments, builder=None, name=None, force_file=False): """ Creates a response-file with the specified *name* in the in the ``buildfiles/`` directory and writes the *arguments* list quoted into the file. If *builder* is specified, it must be a :class:`TargetBuilder` and the response file will be added to the implicit dependencies. If *force_file* is set to True, a file will always be written. Otherwise, the function will into possible limitations of the platform and decide whether to write a response file or to return the *arguments* as is. Returns a tuple of ``(filename, arguments)``. If a response file is written, the returned *arguments* will be a list with a single string that is the filename prepended with ``@``. The *filename* part can be None if no response file needed to be exported. """ if not name: if not builder: raise ValueError('builder must be specified if name is bot') name = builder.name + '.response.txt' if platform.name != 'win': return None, arguments # We'll just assume that there won't be more than 2048 characters for # other flags. The windows max buffer size is 8192. content = shell.join(arguments) if len(content) < 6144: return None, arguments filename = buildlocal(path.join('buildfiles', name)) if builder: builder.implicit_deps.append(filename) if session.builddir: path.makedirs(path.dirname(filename)) with open(filename, 'w') as fp: fp.write(content) return filename, ['@' + filename]
def write_command_file(self, filename, commands, inputs=None, outputs=None, cwd=None, environ=None, foreach=False, suffix='.sh', dry=False, accept_additional_args=False): if suffix is not None: filename = path.addsuffix(filename, suffix) result = [filename] if foreach: result += ['$in', '$out'] inputs, outputs = ['%1'], ['%2'] commands = self.replace_commands_inout_vars(commands, inputs, outputs) if dry: return result, filename path.makedirs(path.dirname(filename)) with open(filename, 'w') as fp: # TODO: Make sure this also works for shells other than bash. fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) + '\n') fp.write('set -e\n') if cwd: fp.write('cd ' + shell.quote(cwd) + '\n') fp.write('\n') for key, value in environ.items(): fp.write('export {}={}\n'.format(key, shell.quote(value))) fp.write('\n') for index, command in enumerate(commands): if accept_additional_args and index == len(commands)-1: command.append(shell.safe('$*')) fp.write(shell.join(command)) fp.write('\n') os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # rwxrw-r-- return result, filename
def download_file(url, filename=None, file=None, directory=None, on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None): """ Download a file from a URL to one of the following destinations: :param filename: A filename to write the downloaded file to. :param file: A file-like object. :param directory: A directory. The filename will be automatically determined from the ``Content-Disposition`` header received from the server or the last path elemnet in the URL. Additional parameters for the *directory* parameter: :param on_exists: The operation to perform when the file already exists. Available modes are ``rename``, ``overwrite`` and ``skip``. Additional parameters: :param progress: A callable that accepts a single parameter that is a dictionary with information about the progress of the download. The dictionary provides the keys ``size``, ``downloaded`` and ``response``. If the callable returns :const:`False` (specifically the value False), the download will be aborted and a :class:`UserInterrupt` will be raised. :param urlopen_kwargs: A dictionary with additional keyword arguments for :func:`urllib.request.urlopen`. Raise and return: :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`. :raise URLError: Can be raised by :func:`urllib.request.urlopen`. :raise UserInterrupt: If the *progress* returned :const:`False`. :return: If the download mode is *directory*, the name of the downloaded file will be returned and False if the file was newly downloaded, True if the download was skipped because the file already existed. Otherwise, the number of bytes downloaded will be returned. """ argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']}) if sum(map(bool, [filename, file, directory])) != 1: raise ValueError( 'exactly one of filename, file or directory must be specifed') response = urllib.request.urlopen(url, **(urlopen_kwargs or {})) if directory: try: filename = parse_content_disposition( response.headers.get('Content-Disposition', '')) except ValueError: filename = url.split('/')[-1] filename = path.join(directory, filename) path.makedirs(directory) if path.exists(filename): if on_exists == 'skip': return filename, True elif on_exists == 'rename': index = 0 while True: new_filename = filename + '_{:0>4}'.format(index) if not path.exists(new_filename): filename = new_filename break index += 1 elif on_exists != 'overwrite': raise RuntimeError try: size = int(response.headers.get('Content-Length', '')) except ValueError: size = None progress_info = { 'response': response, 'size': size, 'downloaded': 0, 'completed': False, 'filename': filename, 'url': url } if progress and progress(progress_info) is False: raise UserInterrupt def copy_to_file(fp): while True: data = response.read(chunksize) if not data: break progress_info['downloaded'] += len(data) fp.write(data) if progress and progress(progress_info) is False: raise UserInterrupt progress_info['completed'] = True if progress and progress(progress_info) is False: raise UserInterrupt if filename: path.makedirs(path.dirname(filename)) try: with open(filename, 'wb') as fp: copy_to_file(fp) except BaseException: # Delete the file if it could not be downloaded successfully. path.remove(filename, silent=True) raise elif file: copy_to_file(file) return filename, False
def download_file(url, filename=None, file=None, directory=None, on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None): """ Download a file from a URL to one of the following destinations: :param filename: A filename to write the downloaded file to. :param file: A file-like object. :param directory: A directory. The filename will be automatically determined from the ``Content-Disposition`` header received from the server or the last path elemnet in the URL. Additional parameters for the *directory* parameter: :param on_exists: The operation to perform when the file already exists. Available modes are ``rename``, ``overwrite`` and ``skip``. Additional parameters: :param progress: A callable that accepts a single parameter that is a dictionary with information about the progress of the download. The dictionary provides the keys ``size``, ``downloaded`` and ``response``. If the callable returns :const:`False` (specifically the value False), the download will be aborted and a :class:`UserInterrupt` will be raised. :param urlopen_kwargs: A dictionary with additional keyword arguments for :func:`urllib.request.urlopen`. Raise and return: :raise HTTPError: Can be raised by :func:`urllib.request.urlopen`. :raise URLError: Can be raised by :func:`urllib.request.urlopen`. :raise UserInterrupt: If the *progress* returned :const:`False`. :return: If the download mode is *directory*, the name of the downloaded file will be returned and False if the file was newly downloaded, True if the download was skipped because the file already existed. Otherwise, the number of bytes downloaded will be returned. """ argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']}) if sum(map(bool, [filename, file, directory])) != 1: raise ValueError('exactly one of filename, file or directory must be specifed') response = urllib.request.urlopen(url, **(urlopen_kwargs or {})) if directory: try: filename = parse_content_disposition( response.headers.get('Content-Disposition', '')) except ValueError: filename = url.split('/')[-1] filename = path.join(directory, filename) path.makedirs(directory) if path.exists(filename): if on_exists == 'skip': return filename, True elif on_exists == 'rename': index = 0 while True: new_filename = filename + '_{:0>4}'.format(index) if not path.exists(new_filename): filename = new_filename break index += 1 elif on_exists != 'overwrite': raise RuntimeError try: size = int(response.headers.get('Content-Length', '')) except ValueError: size = None progress_info = {'response': response, 'size': size, 'downloaded': 0, 'completed': False, 'filename': filename, 'url': url} if progress and progress(progress_info) is False: raise UserInterrupt def copy_to_file(fp): while True: data = response.read(chunksize) if not data: break progress_info['downloaded'] += len(data) fp.write(data) if progress and progress(progress_info) is False: raise UserInterrupt progress_info['completed'] = True if progress and progress(progress_info) is False: raise UserInterrupt if filename: path.makedirs(path.dirname(filename)) try: with open(filename, 'wb') as fp: copy_to_file(fp) except BaseException: # Delete the file if it could not be downloaded successfully. path.remove(filename, silent=True) raise elif file: copy_to_file(file) return filename, False
def execute(self, parser, args): session.path.extend(map(path.norm, args.include_path)) if self.is_export: # Determine the module to execute, either from the current working # directory or find it by name if one is specified. if not args.module: for fn in [MANIFEST_FILENAME, path.join('craftr', MANIFEST_FILENAME)]: if path.isfile(fn): module = session.parse_manifest(fn) break else: parser.error('"{}" does not exist'.format(MANIFEST_FILENAME)) else: # TODO: For some reason, prints to stdout are not visible here. # TODO: Prints to stderr however work fine. try: module_name, version = parse_module_spec(args.module) except ValueError as exc: parser.error('{} (note: you have to escape > and < characters)'.format(exc)) try: module = session.find_module(module_name, version) except Module.NotFound as exc: parser.error('module not found: ' + str(exc)) else: module = None ninja_bin, ninja_version = get_ninja_info() # Create and switch to the build directory. session.builddir = path.abs(args.build_dir) path.makedirs(session.builddir) os.chdir(session.builddir) # Read the cache and parse command-line options. cachefile = path.join(session.builddir, '.craftrcache') if not read_cache(cachefile) and not self.is_export: logger.error('Unable to load "{}", can not build'.format(cachefile)) return 1 # Prepare options, loaders and execute. if self.is_export: session.cache['build'] = {} try: write_cache(cachefile) module.run() except (Module.InvalidOption, Module.LoaderInitializationError) as exc: for error in exc.format_errors(): logger.error(error) return 1 except craftr.defaults.ModuleError as exc: logger.error(exc) return 1 # Write the cache back. session.cache['build']['targets'] = list(session.graph.targets.keys()) session.cache['build']['main'] = module.ident session.cache['build']['options'] = args.options write_cache(cachefile) # Write the Ninja manifest. with open("build.ninja", 'w') as fp: platform = core.build.get_platform_helper() context = core.build.ExportContext(ninja_version) writer = core.build.NinjaWriter(fp) session.graph.export(writer, context, platform) else: parse_cmdline_options(session.cache['build']['options']) main = session.cache['build']['main'] available_targets = frozenset(session.cache['build']['targets']) # Check the targets and if they exist. targets = [] for target in args.targets: if '.' not in target: target = main + '.' + target elif target.startswith('.'): target = main + target module_name, target = target.rpartition('.')[::2] module_name, version = get_volatile_module_version(module_name) ref_module = session.find_module(module_name, version or '*') target = craftr.targetbuilder.get_full_name(target, ref_module) if target not in available_targets: parser.error('no such target: {}'.format(target)) targets.append(target) # Execute the ninja build. cmd = [ninja_bin] if args.verbose: cmd += ['-v'] cmd += targets shell.run(cmd)
def execute(self, parser, args): session.path.extend(map(path.norm, args.include_path)) if self.mode == "export": # Determine the module to execute, either from the current working # directory or find it by name if one is specified. if not args.module: for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]: if path.isfile(fn): module = session.parse_manifest(fn) break else: parser.error('"{}" does not exist'.format(MANIFEST_FILENAME)) else: # TODO: For some reason, prints to stdout are not visible here. # TODO: Prints to stderr however work fine. try: module_name, version = parse_module_spec(args.module) except ValueError as exc: parser.error("{} (note: you have to escape > and < characters)".format(exc)) try: module = session.find_module(module_name, version) except Module.NotFound as exc: parser.error("module not found: " + str(exc)) else: module = None ninja_bin, ninja_version = get_ninja_info() # Create and switch to the build directory. session.builddir = path.abs(args.build_dir) path.makedirs(session.builddir) os.chdir(session.builddir) # Read the cache and parse command-line options. cachefile = path.join(session.builddir, ".craftrcache") if not read_cache(cachefile) and self.mode != "export": logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode)) logger.error("Make sure to generate a build tree with 'craftr export'") return 1 # Prepare options, loaders and execute. if self.mode == "export": session.expand_relative_options(module.manifest.name) session.cache["build"] = {} try: module.run() except Module.InvalidOption as exc: for error in exc.format_errors(): logger.error(error) return 1 except craftr.defaults.ModuleError as exc: logger.error("error:", exc) return 1 finally: if sys.exc_info(): # We still want to write the cache, especially so that data already # loaded with loaders doesn't need to be re-loaded. They'll find out # when the cached information was not valid. write_cache(cachefile) # Write the cache back. session.cache["build"]["targets"] = list(session.graph.targets.keys()) session.cache["build"]["main"] = module.ident session.cache["build"]["options"] = args.options write_cache(cachefile) # Write the Ninja manifest. with open("build.ninja", "w") as fp: platform = core.build.get_platform_helper() context = core.build.ExportContext(ninja_version) writer = core.build.NinjaWriter(fp) session.graph.export(writer, context, platform) else: parse_cmdline_options(session.cache["build"]["options"]) main = session.cache["build"]["main"] available_targets = frozenset(session.cache["build"]["targets"]) logger.debug("build main module:", main) session.expand_relative_options(get_volatile_module_version(main)[0]) # Check the targets and if they exist. targets = [] for target in args.targets: if "." not in target: target = main + "." + target elif target.startswith("."): target = main + target module_name, target = target.rpartition(".")[::2] module_name, version = get_volatile_module_version(module_name) ref_module = session.find_module(module_name, version or "*") target = craftr.targetbuilder.get_full_name(target, ref_module) if target not in available_targets: parser.error("no such target: {}".format(target)) targets.append(target) # Execute the ninja build. cmd = [ninja_bin] if args.verbose: cmd += ["-v"] if self.mode == "clean": cmd += ["-t", "clean"] if not args.recursive: cmd += ["-r"] cmd += targets return shell.run(cmd).returncode