def parse_manifest(self, filename): """ Parse a manifest by filename and add register the module to the module cache. Returns the :class:`Module` object. If the manifest has already been parsed, it will not be re-parsed. :raise Manifest.Invalid: If the manifest is invalid. :return: :const:`None` if the manifest is a duplicate of an already parsed manifest (determined by name and version), otherwise the :class:`Module` object for the manifest's module. """ filename = path.norm(path.abs(filename)) if filename in self._manifest_cache: manifest = self._manifest_cache[filename] return self.find_module(manifest.name, manifest.version) manifest = Manifest.parse(filename) self._manifest_cache[filename] = manifest versions = self.modules.setdefault(manifest.name, {}) if manifest.version in versions: logger.debug('multiple occurences of "{}-{}" found, ' 'one of which is located at "{}"'.format(manifest.name, manifest.version, filename)) module = None else: logger.debug('parsed manifest: {}-{} ({})'.format( manifest.name, manifest.version, filename)) module = Module(path.dirname(filename), manifest) versions[manifest.version] = module return module
def parse_manifest(self, filename): """ Parse a manifest by filename and add register the module to the module cache. Returns the :class:`Module` object. If the manifest has already been parsed, it will not be re-parsed. :raise Manifest.Invalid: If the manifest is invalid. :return: :const:`None` if the manifest is a duplicate of an already parsed manifest (determined by name and version), otherwise the :class:`Module` object for the manifest's module. """ filename = path.norm(path.abs(filename)) if filename in self._manifest_cache: manifest = self._manifest_cache[filename] return self.find_module(manifest.name, manifest.version) manifest = Manifest.parse(filename) self._manifest_cache[filename] = manifest versions = self.modules.setdefault(manifest.name, {}) if manifest.version in versions: other = versions[manifest.version].manifest.filename logger.debug('multiple occurences of "{}-{}" found\n' ' - {}\n - {}'.format(manifest.name, manifest.version, filename, other)) module = None else: logger.debug('parsed manifest: {}-{} ({})'.format( manifest.name, manifest.version, filename)) module = Module(path.dirname(filename), manifest) versions[manifest.version] = module return module
def get_ninja_info(): # Make sure the Ninja executable exists and find its version. ninja_bin = session.options.get('global.ninja') or \ session.options.get('craftr.ninja') or os.getenv('NINJA', 'ninja') ninja_bin = shell.find_program(ninja_bin) ninja_version = get_ninja_version(ninja_bin) logger.debug('Ninja executable:', ninja_bin) logger.debug('Ninja version:', ninja_version) return ninja_bin, ninja_version
def find_module(self, name, version, resolve_preferred_version=True): """ Finds a module in the :attr:`path` matching the specified *name* and *version*. :param name: The name of the module. :param version: A :class:`VersionCriteria`, :class:`Version` or string in a VersionCritiera format. :param resolve_preferred_version: If this parameter is True (default) and a preferred version is specified in :attr:`preferred_versions`, that preferred version is loaded or :class:`ModuleNotFound` is raised. :raise ModuleNotFound: If the module can not be found. :return: :class:`Module` """ argspec.validate('name', name, {'type': str}) argspec.validate('version', version, {'type': [str, Version, VersionCriteria]}) if name in renames.renames: logger.warn('"{}" is deprecated, use "{}" instead'.format( name, renames.renames[name])) name = renames.renames[name] if isinstance(version, str): try: version = Version(version) except ValueError as exc: version = VersionCriteria(version) if session.module and resolve_preferred_version: data = self.preferred_versions.get(session.module.manifest.name) if data is not None: versions = data.get(str(session.module.manifest.version)) if versions is not None: preferred_version = versions.get(name) if preferred_version is not None: version = Version(preferred_version) logger.debug( 'note: loading preferred version {} of module "{}" ' 'requested by module "{}"'.format( version, name, session.module.ident)) self.update_manifest_cache() if name in self.modules: if isinstance(version, Version): if version in self.modules[name]: return self.modules[name][version] raise ModuleNotFound(name, version) for module in sorted(self.modules[name].values(), key=lambda x: x.manifest.version, reverse=True): if version(module.manifest.version): return module raise ModuleNotFound(name, version)
def get_ninja_info(): # Make sure the Ninja executable exists and find its version. ninja_bin = ( session.options.get("global.ninja") or session.options.get("craftr.ninja") or os.getenv("NINJA", "ninja") ) ninja_bin = shell.find_program(ninja_bin) ninja_version = get_ninja_version(ninja_bin) logger.debug("Ninja executable:", ninja_bin) logger.debug("Ninja version:", ninja_version) return ninja_bin, ninja_version
def write_cache(cachefile): # Write back the cache. try: path.makedirs(path.dirname(cachefile)) with open(cachefile, 'w') as fp: session.write_cache(fp) except OSError as exc: logger.error('error writing cache file:', cachefile) logger.error(exc, indent=1) else: logger.debug('cache written:', cachefile)
def __exit__(self, exc_value, exc_type, exc_tb): if Session.current is not self: raise RuntimeError('session not in context') if self._tempdir and not self.options.get('craftr.keep_temporary_directory'): logger.debug('removing temporary directory:', self._tempdir) try: path.remove(self._tempdir, recursive=True) except OSError as exc: logger.debug('error:', exc, indent=1) finally: self._tempdir = None Session.current = None
def __cleanup(self, parser, args): """ Switch back to the original directory and check if we can clean up the build directory. """ os.chdir(session.maindir) if os.path.isdir( session.builddir) and not os.listdir(session.builddir): logger.debug('note: cleanup empty build directory:', session.builddir) os.rmdir(session.builddir)
def __exit__(self, exc_value, exc_type, exc_tb): if Session.current is not self: raise RuntimeError('session not in context') if self._tempdir and not self.options.get( 'craftr.keep_temporary_directory'): logger.debug('removing temporary directory:', self._tempdir) try: path.remove(self._tempdir, recursive=True) except OSError as exc: logger.debug('error:', exc, indent=1) finally: self._tempdir = None Session.current = None
def get_temporary_directory(self): """ Returns a writable temporary directory that is primarily used by loaders to store temporary files. The temporary directory will be deleted when the Session context ends unless the ``craftr.keep_temporary_directory`` option is set. :raise RuntimeError: If the session is not currently in context. """ if Session.current is not self: raise RuntimeError('session not in context') if not self._tempdir: self._tempdir = path.join(self.builddir, '.temp') logger.debug('created temporary directory:', self._tempdir) return self._tempdir
def execute(self, parser, args): directory = args.directory or args.name if path.maybedir(directory): directory = path.join(directory, args.name) if not path.exists(directory): logger.debug('creating directory "{}"'.format(directory)) path.makedirs(directory) elif not path.isdir(directory): logger.error('"{}" is not a directory'.format(directory)) return 1 if args.nested: directory = path.join(directory, 'craftr') path.makedirs(directory) mfile = path.join(directory, 'manifest.' + args.format) sfile = path.join(directory, 'Craftrfile') for fn in [mfile, sfile]: if path.isfile(fn): logger.error('"{}" already exists'.format(fn)) return 1 logger.debug('creating file "{}"'.format(mfile)) with open(mfile, 'w') as fp: if args.format == 'cson': lines = textwrap.dedent(''' name: "%s" version: "%s" project_dir: ".." author: "" url: "" dependencies: {} options: {} ''' % (args.name, args.version)).lstrip().split('\n') if not args.nested: del lines[2] elif args.format == 'json': lines = textwrap.dedent(''' { "name": "%s", "version": "%s", "project_dir": "..", "author": "", "url": "", "dependencies": {}, "options": {} }''' % (args.name, args.version)).lstrip().split('\n') if not args.nested: del lines[3] fp.write('\n'.join(lines)) logger.debug('creating file "{}"'.format(sfile)) with open(sfile, 'w') as fp: print('# {}'.format(args.name), file=fp)
def execute(self, parser, args): directory = args.directory or args.name if path.maybedir(directory): directory = path.join(directory, args.name) if not path.exists(directory): logger.debug('creating directory "{}"'.format(directory)) path.makedirs(directory) elif not path.isdir(directory): logger.error('"{}" is not a directory'.format(directory)) return 1 if args.nested: directory = path.join(directory, "craftr") path.makedirs(directory) mfile = path.join(directory, MANIFEST_FILENAME) sfile = path.join(directory, "Craftrfile") for fn in [mfile, sfile]: if path.isfile(fn): logger.error('"{}" already exists'.format(fn)) return 1 logger.debug('creating file "{}"'.format(mfile)) with open(mfile, "w") as fp: lines = ( textwrap.dedent( """ { "name": "%s", "version": "%s", "project_dir": "..", "author": "", "url": "", "dependencies": {}, "options": {} }\n""" % (args.name, args.version) ) .lstrip() .split("\n") ) if not args.nested: del lines[3] fp.write("\n".join(lines)) logger.debug('creating file "{}"'.format(sfile)) with open(sfile, "w") as fp: print("# {}".format(args.name), file=fp)
def read_config_file(filename, basedir=None, follow_include_directives=True): """ Reads a configuration file and returns a dictionary of the values that it contains. The format is standard :mod:`configparser` ``.ini`` style, however this function supports ``include`` directives that can include additional configuration files. :: [include "path/to/config.ini"] ; errors if the file does not exist [include "path/to/config.ini" if-exists] ; ignored if the file does not exist :param filename: The name of the configuration file to read. :param basedir: If *filename* is not an absolute path or the base directory should be altered, this is the directory of which to look for files specified with ``include`` directives. :param follow_include_directives: If this is True, ``include`` directives will be followed. :raise FileNotFoundError: If *filename* does not exist. :raise InvalidConfigError: If the configuration format is invalid. Also if any of the included files do not exist. :return: A dictionary. Section names are prepended to the option names. """ filename = path.norm(filename) if not basedir: basedir = path.dirname(filename) if not path.isfile(filename): raise FileNotFoundError(filename) logger.debug('reading configuration file:', filename) parser = configparser.SafeConfigParser() try: parser.read([filename]) except configparser.Error as exc: raise InvalidConfigError('"{}": {}'.format(filename, exc)) result = {} for section in parser.sections(): match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section) if match: if not follow_include_directives: continue ifile, if_exists = match.groups() ifile = path.norm(ifile, basedir) try: result.update(read_config_file(ifile)) except FileNotFoundError as exc: if not if_exists: raise InvalidConfigError( 'file "{}" included by "{}" does not exist'.format( str(exc), filename)) continue elif section == '__global__': prefix = '' else: prefix = section + '.' for option in parser.options(section): result[prefix + option] = parser.get(section, option) return result
def _build_or_clean(self, args): """ Will be called for the 'build' and 'clean' modes. Loads the Craftr cache and invokes Ninja. """ # Read the cache and parse command-line options. if not read_cache(True): sys.exit(1) parse_cmdline_options(session.cache['build']['options']) main = session.cache['build']['main'] available_targets = frozenset(session.cache['build']['targets']) available_modules = unserialise_loaded_module_info( session.cache['build']['modules']) logger.debug('build main module:', main) session.expand_relative_options(get_volatile_module_version(main)[0]) # Check if any of the modules changed, so we can let the user know he # might have to re-export the build files. changed_modules = [] for name, versions in available_modules.items(): for version, info in versions.items(): if info['changed']: changed_modules.append('{}-{}'.format(name, version)) if changed_modules: if len(changed_modules) == 1: logger.info( 'note: module "{}" has changed, maybe you should re-export' .format(changed_modules[0])) else: logger.info( 'note: some modules have changed, maybe you should re-export' ) for name in changed_modules: logger.info(' -', name) # Check the targets and if they exist. targets = [] for target_name in args.targets: if '.' not in target_name: target_name = main + '.' + target_name elif target_name.startswith('.'): target_name = main + target_name module_name, target_name = target_name.rpartition('.')[::2] module_name, version = get_volatile_module_version(module_name) if module_name not in available_modules: error('no such module:', module_name) if not version: version = max(available_modules[module_name].keys()) target_name = craftr.targetbuilder.get_full_name( target_name, module_name=module_name, version=version) if target_name not in available_targets: logger.error('no such target: {}'.format(target_name)) return 1 targets.append(target_name) # Make sure we get all the output before running the subcommand. logger.flush() # Execute the ninja build. cmd = [self.ninja_bin] if args.verbose: cmd += ['-v'] if self.mode == 'clean': cmd += ['-t', 'clean'] if not args.recursive: cmd += ['-r'] cmd += targets return shell.run(cmd).returncode
def _export_run_or_help(self, args, module): """ Called when the mode is 'export' or 'run'. Will execute the specified *module* and eventually export a Ninja manifest and Cache. """ read_cache(False) session.expand_relative_options() session.cache['build'] = {} # Load the dependency lock information if it exists. deplock_fn = path.join(path.dirname(module.manifest.filename), '.dependency-lock') if os.path.isfile(deplock_fn): with open(deplock_fn) as fp: session.preferred_versions = cson.load(fp) logger.debug('note: dependency lock file "{}" loaded'.format( deplock_fn)) try: module.run() except Module.InvalidOption as exc: for error in exc.format_errors(): logger.error(error) return 1 except craftr.defaults.ModuleError as exc: logger.error('error:', exc) return 1 finally: if sys.exc_info() and self.mode == 'export': # We still want to write the cache, especially so that data already # loaded with loaders doesn't need to be re-loaded. They'll find out # when the cached information was not valid. write_cache(self.cachefile) # Fill the cache. session.cache['build']['targets'] = list(session.graph.targets.keys()) session.cache['build']['modules'] = serialise_loaded_module_info() session.cache['build']['main'] = module.ident session.cache['build']['options'] = args.options session.cache['build']['dependency_lock_filename'] = deplock_fn if self.mode == 'export': # Add the Craftr_run_command variable which is necessary for tasks # to properly executed. run_command = ['craftr', '-q', '-P', path.rel(session.maindir)] if args.no_config: run_command += ['-C'] run_command += ['-c' + x for x in args.config] run_command += ['run'] if args.module: run_command += ['-m', args.module] run_command += ['-i' + x for x in args.include_path] run_command += ['-b', path.rel(session.builddir)] session.graph.vars['Craftr_run_command'] = shell.join(run_command) write_cache(self.cachefile) # Write the Ninja manifest. with open("build.ninja", 'w') as fp: platform = core.build.get_platform_helper() context = core.build.ExportContext(self.ninja_version) writer = core.build.NinjaWriter(fp) session.graph.export(writer, context, platform) logger.info('exported "build.ninja"') return 0 elif self.mode == 'run': if args.task: if args.task not in session.graph.tasks: logger.error('no such task exists: "{}"'.format(args.task)) return 1 task = session.graph.tasks[args.task] return task.invoke(args.task_args) return 0 elif self.mode == 'help': if args.name not in vars(module.namespace): logger.error('symbol not found: "{}:{}"'.format( module.manifest.name, args.name)) return 1 help(getattr(module.namespace, args.name)) return 0 assert False, "unhandled mode: {}".format(self.mode)
def execute(self, parser, args): session.path.extend(map(path.norm, args.include_path)) if self.mode == "export": # Determine the module to execute, either from the current working # directory or find it by name if one is specified. if not args.module: for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]: if path.isfile(fn): module = session.parse_manifest(fn) break else: parser.error('"{}" does not exist'.format(MANIFEST_FILENAME)) else: # TODO: For some reason, prints to stdout are not visible here. # TODO: Prints to stderr however work fine. try: module_name, version = parse_module_spec(args.module) except ValueError as exc: parser.error("{} (note: you have to escape > and < characters)".format(exc)) try: module = session.find_module(module_name, version) except Module.NotFound as exc: parser.error("module not found: " + str(exc)) else: module = None ninja_bin, ninja_version = get_ninja_info() # Create and switch to the build directory. session.builddir = path.abs(args.build_dir) path.makedirs(session.builddir) os.chdir(session.builddir) # Read the cache and parse command-line options. cachefile = path.join(session.builddir, ".craftrcache") if not read_cache(cachefile) and self.mode != "export": logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode)) logger.error("Make sure to generate a build tree with 'craftr export'") return 1 # Prepare options, loaders and execute. if self.mode == "export": session.expand_relative_options(module.manifest.name) session.cache["build"] = {} try: module.run() except Module.InvalidOption as exc: for error in exc.format_errors(): logger.error(error) return 1 except craftr.defaults.ModuleError as exc: logger.error("error:", exc) return 1 finally: if sys.exc_info(): # We still want to write the cache, especially so that data already # loaded with loaders doesn't need to be re-loaded. They'll find out # when the cached information was not valid. write_cache(cachefile) # Write the cache back. session.cache["build"]["targets"] = list(session.graph.targets.keys()) session.cache["build"]["main"] = module.ident session.cache["build"]["options"] = args.options write_cache(cachefile) # Write the Ninja manifest. with open("build.ninja", "w") as fp: platform = core.build.get_platform_helper() context = core.build.ExportContext(ninja_version) writer = core.build.NinjaWriter(fp) session.graph.export(writer, context, platform) else: parse_cmdline_options(session.cache["build"]["options"]) main = session.cache["build"]["main"] available_targets = frozenset(session.cache["build"]["targets"]) logger.debug("build main module:", main) session.expand_relative_options(get_volatile_module_version(main)[0]) # Check the targets and if they exist. targets = [] for target in args.targets: if "." not in target: target = main + "." + target elif target.startswith("."): target = main + target module_name, target = target.rpartition(".")[::2] module_name, version = get_volatile_module_version(module_name) ref_module = session.find_module(module_name, version or "*") target = craftr.targetbuilder.get_full_name(target, ref_module) if target not in available_targets: parser.error("no such target: {}".format(target)) targets.append(target) # Execute the ninja build. cmd = [ninja_bin] if args.verbose: cmd += ["-v"] if self.mode == "clean": cmd += ["-t", "clean"] if not args.recursive: cmd += ["-r"] cmd += targets return shell.run(cmd).returncode
def read_config_file(filename, basedir=None, follow_include_directives=True): """ Reads a configuration file and returns a dictionary of the values that it contains. The format is standard :mod:`configparser` ``.ini`` style, however this function supports ``include`` directives that can include additional configuration files. :: [include "path/to/config.ini"] ; errors if the file does not exist [include "path/to/config.ini" if-exists] ; ignored if the file does not exist :param filename: The name of the configuration file to read. :param basedir: If *filename* is not an absolute path or the base directory should be altered, this is the directory of which to look for files specified with ``include`` directives. :param follow_include_directives: If this is True, ``include`` directives will be followed. :raise FileNotFoundError: If *filename* does not exist. :raise InvalidConfigError: If the configuration format is invalid. Also if any of the included files do not exist. :return: A dictionary. Section names are prepended to the option names. """ filename = path.norm(filename) if not basedir: basedir = path.dirname(filename) if not path.isfile(filename): raise FileNotFoundError(filename) logger.debug("reading configuration file:", filename) parser = configparser.SafeConfigParser() try: parser.read([filename]) except configparser.Error as exc: raise InvalidConfigError('"{}": {}'.format(filename, exc)) result = {} for section in parser.sections(): match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section) if match: if not follow_include_directives: continue ifile, if_exists = match.groups() ifile = path.norm(ifile, basedir) try: result.update(read_config_file(ifile)) except FileNotFoundError as exc: if not if_exists: raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename)) continue elif section == "__global__": prefix = "" else: prefix = section + "." for option in parser.options(section): result[prefix + option] = parser.get(section, option) return result
def load(self, context, cache): if cache is not None and path.isdir(cache.get("directory", "")): # Check if the requested version changes. url_template = context.expand_variables(cache.get("url_template", "")) if url_template == cache.get("url"): self.directory = cache["directory"] logger.info("Reusing cached directory: {}".format(path.rel(self.directory, nopar=True))) return cache else: logger.info("Cached URL is outdated:", cache.get("url")) directory = None archive = None delete_after_extract = True for url_template in self.urls: url = context.expand_variables(url_template) if not url: continue if url.startswith("file://"): name = url[7:] if path.isdir(name): logger.info("Using directory", url) directory = name break elif path.isfile(name): logger.info("Using archive", url) archive = name delete_after_extract = False break error = None else: error = None try: progress = lambda d: self._download_progress(url, context, d) archive, reused = httputils.download_file( url, directory=context.get_temporary_directory(), on_exists="skip", progress=progress ) except (httputils.URLError, httputils.HTTPError) as exc: error = exc except self.DownloadAlreadyExists as exc: directory = exc.directory logger.info("Reusing existing directory", directory) else: if reused: logger.info("Reusing cached download", path.basename(archive)) break if error: logger.info("Error reading", url, ":", error) if directory or archive: logger.debug("URL applies: {}".format(url)) if not directory and archive: suffix, directory = self._get_archive_unpack_info(context, archive) logger.info( 'Unpacking "{}" to "{}" ...'.format(path.rel(archive, nopar=True), path.rel(directory, nopar=True)) ) nr.misc.archive.extract( archive, directory, suffix=suffix, unpack_single_dir=True, check_extract_file=self._check_extract_file, progress_callback=self._extract_progress, ) elif not directory: raise LoaderError(self, "no URL matched") self.directory = directory with open(path.join(self.directory, ".craftr_downloadurl"), "w") as fp: fp.write(url) return {"directory": directory, "url_template": url_template, "url": url}