def parse_manifest(self, filename): """ Parse a manifest by filename and add register the module to the module cache. Returns the :class:`Module` object. If the manifest has already been parsed, it will not be re-parsed. :raise Manifest.Invalid: If the manifest is invalid. :return: :const:`None` if the manifest is a duplicate of an already parsed manifest (determined by name and version), otherwise the :class:`Module` object for the manifest's module. """ filename = path.norm(path.abs(filename)) if filename in self._manifest_cache: manifest = self._manifest_cache[filename] return self.find_module(manifest.name, manifest.version) manifest = Manifest.parse(filename) self._manifest_cache[filename] = manifest versions = self.modules.setdefault(manifest.name, {}) if manifest.version in versions: logger.debug('multiple occurences of "{}-{}" found, ' 'one of which is located at "{}"'.format(manifest.name, manifest.version, filename)) module = None else: logger.debug('parsed manifest: {}-{} ({})'.format( manifest.name, manifest.version, filename)) module = Module(path.dirname(filename), manifest) versions[manifest.version] = module return module
def load_file(filename, export_default_namespace=True): """ Loads a Python file into a new module-like object and returns it. The *filename* is assumed relative to the currently executed module's directory (NOT the project directory which can be different). """ module = session.module __name__ = module.ident + ':' + filename if not path.isabs(filename): filename = path.join(module.directory, filename) filename = path.norm(filename) with open(filename, 'r') as fp: code = compile(fp.read(), filename, 'exec') scope = Namespace() if export_default_namespace: vars(scope).update(module.get_init_globals()) scope.__module__ = module.namespace scope.__file__ = filename scope.__name__ = __name__ exec(code, vars(scope)) return scope
def parse_manifest(self, filename): """ Parse a manifest by filename and add register the module to the module cache. Returns the :class:`Module` object. If the manifest has already been parsed, it will not be re-parsed. :raise Manifest.Invalid: If the manifest is invalid. :return: :const:`None` if the manifest is a duplicate of an already parsed manifest (determined by name and version), otherwise the :class:`Module` object for the manifest's module. """ filename = path.norm(path.abs(filename)) if filename in self._manifest_cache: manifest = self._manifest_cache[filename] return self.find_module(manifest.name, manifest.version) manifest = Manifest.parse(filename) self._manifest_cache[filename] = manifest versions = self.modules.setdefault(manifest.name, {}) if manifest.version in versions: other = versions[manifest.version].manifest.filename logger.debug('multiple occurences of "{}-{}" found\n' ' - {}\n - {}'.format(manifest.name, manifest.version, filename, other)) module = None else: logger.debug('parsed manifest: {}-{} ({})'.format( manifest.name, manifest.version, filename)) module = Module(path.dirname(filename), manifest) versions[manifest.version] = module return module
def load_file(filename, export_default_namespace=True): """ Loads a Python file into a new module-like object and returns it. The *filename* is assumed relative to the currently executed module's directory (NOT the project directory which can be different). """ module = session.module __name__ = module.ident + ':' + filename if not path.isabs(filename): filename = path.join(module.directory, filename) filename = path.norm(filename) module.dependent_files.append(filename) with open(filename, 'r') as fp: code = compile(fp.read(), filename, 'exec') scope = Namespace() if export_default_namespace: vars(scope).update(module.get_init_globals()) scope.__module__ = module.namespace scope.__file__ = filename scope.__name__ = __name__ exec(code, vars(scope)) return scope
def run(self): """ Loads the code of the main Craftr build script as specified in the modules manifest and executes it. Note that this must occur in a context where the :data:`session` is available. :raise RuntimeError: If there is no current :data:`session` or if the module was already executed. """ if not session: raise RuntimeError('no current session') if self.executed: raise RuntimeError('already run') self.executed = True self.init_options() script_fn = path.norm(path.join(self.directory, self.manifest.main)) with open(script_fn) as fp: code = compile(fp.read(), script_fn, 'exec') vars(self.namespace).update(self.get_init_globals()) self.namespace.__file__ = script_fn self.namespace.__name__ = self.manifest.name self.namespace.__version__ = str(self.manifest.version) try: session.modulestack.append(self) exec(code, vars(self.namespace)) finally: assert session.modulestack.pop() is self
def local(rel_path): """ Given a relative path, returns the absolute path relative to the current module's project directory. """ parent = session.module.project_dir return path.norm(rel_path, parent)
def local(rel_path): """ Given a relative path, returns the absolute path relative to the current module's project directory. """ parent = session.module.namespace.project_dir return path.norm(rel_path, parent)
def __init__(self, maindir=None): self.maindir = path.norm(maindir or path.getcwd()) self.builddir = path.join(self.maindir, 'build') self.graph = build.Graph() self.path = [self.stl_dir, self.maindir, path.join(self.maindir, 'craftr/modules')] self.modulestack = [] self.modules = {} self.options = {} self.cache = {'loaders': {}} self._tempdir = None self._manifest_cache = {} # maps manifest_filename: manifest self._refresh_cache = True
def __lshift__(self, other): """ Adds *other* as an implicit dependency to the target. :param other: A :class:`Target` or :class:`str`. :return: ``self`` """ if isinstance(other, Target): self.implicit_deps += other.outputs elif isinstance(other, str): self.implicit_deps.append(path.norm(other)) else: raise TypeError("Target.__lshift__() expected Target or str") return self
def __init__(self, maindir=None): self.maindir = path.norm(maindir or path.getcwd()) self.builddir = path.join(self.maindir, 'build') self.graph = build.Graph() self.path = [ self.stl_dir, self.stl_auxiliary_dir, self.maindir, path.join(self.maindir, 'craftr/modules') ] self.modulestack = [] self.modules = {} self.preferred_versions = {} self.main_module = None self.options = {} self.cache = {} self.tasks = {} self._tempdir = None self._manifest_cache = {} # maps manifest_filename: manifest self._refresh_cache = True
def execute(self, parser, args): if hasattr(args, 'include_path'): session.path.extend(map(path.norm, args.include_path)) # Help-command preprocessing. Check if we're to show the help on a builtin # object, otherwise extract the module name if applicable. if self.mode == 'help': if not args.name: help('craftr') return 0 if args.name in vars(craftr.defaults): help(getattr(craftr.defaults, args.name)) return 0 # Check if we have an absolute symbol reference. if ':' in args.name: if args.module: parser.error( '-m/--module option conflicting with name argument: "{}"' .format(args.name)) args.module, args.name = args.name.split(':', 1) module = self._find_module(parser, args) session.main_module = module self.ninja_bin, self.ninja_version = get_ninja_info() # Create and switch to the build directory. session.builddir = path.abs(path.norm(args.build_dir, INIT_DIR)) path.makedirs(session.builddir) os.chdir(session.builddir) self.cachefile = path.join(session.builddir, '.craftrcache') # Prepare options, loaders and execute. if self.mode in ('export', 'run', 'help'): return self._export_run_or_help(args, module) elif self.mode == 'dump-options': return self._dump_options(args, module) elif self.mode == 'dump-deptree': return self._dump_deptree(args, module) elif self.mode in ('build', 'clean'): return self._build_or_clean(args) elif self.mode == 'lock': self._create_lockfile() else: raise RuntimeError("mode: {}".format(self.mode))
def run(self): """ Loads the code of the main Craftr build script as specified in the modules manifest and executes it. Note that this must occur in a context where the :data:`session` is available. :raise RuntimeError: If there is no current :data:`session` or if the module was already executed. """ if not session: raise RuntimeError('no current session') if self.executed: raise RuntimeError('already run') self.executed = True self.init_options() self.init_loader() script_fn = path.norm(path.join(self.directory, self.manifest.main)) with open(script_fn) as fp: code = compile(fp.read(), script_fn, 'exec') from craftr import defaults for key, value in vars(defaults).items(): if not key.startswith('_'): vars(self.namespace)[key] = value vars(self.namespace).update({ '__file__': script_fn, '__name__': self.manifest.name, '__version__': str(self.manifest.version), 'options': self.options, 'loader': self.loader, 'project_dir': self.project_dir, }) try: session.modulestack.append(self) exec(code, vars(self.namespace)) finally: assert session.modulestack.pop() is self
def read_config_file(filename, basedir=None, follow_include_directives=True): """ Reads a configuration file and returns a dictionary of the values that it contains. The format is standard :mod:`configparser` ``.ini`` style, however this function supports ``include`` directives that can include additional configuration files. :: [include "path/to/config.ini"] ; errors if the file does not exist [include "path/to/config.ini" if-exists] ; ignored if the file does not exist :param filename: The name of the configuration file to read. :param basedir: If *filename* is not an absolute path or the base directory should be altered, this is the directory of which to look for files specified with ``include`` directives. :param follow_include_directives: If this is True, ``include`` directives will be followed. :raise FileNotFoundError: If *filename* does not exist. :raise InvalidConfigError: If the configuration format is invalid. Also if any of the included files do not exist. :return: A dictionary. Section names are prepended to the option names. """ filename = path.norm(filename) if not basedir: basedir = path.dirname(filename) if not path.isfile(filename): raise FileNotFoundError(filename) logger.debug("reading configuration file:", filename) parser = configparser.SafeConfigParser() try: parser.read([filename]) except configparser.Error as exc: raise InvalidConfigError('"{}": {}'.format(filename, exc)) result = {} for section in parser.sections(): match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section) if match: if not follow_include_directives: continue ifile, if_exists = match.groups() ifile = path.norm(ifile, basedir) try: result.update(read_config_file(ifile)) except FileNotFoundError as exc: if not if_exists: raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename)) continue elif section == "__global__": prefix = "" else: prefix = section + "." for option in parser.options(section): result[prefix + option] = parser.get(section, option) return result
def read_config_file(filename, basedir=None, follow_include_directives=True): """ Reads a configuration file and returns a dictionary of the values that it contains. The format is standard :mod:`configparser` ``.ini`` style, however this function supports ``include`` directives that can include additional configuration files. :: [include "path/to/config.ini"] ; errors if the file does not exist [include "path/to/config.ini" if-exists] ; ignored if the file does not exist :param filename: The name of the configuration file to read. :param basedir: If *filename* is not an absolute path or the base directory should be altered, this is the directory of which to look for files specified with ``include`` directives. :param follow_include_directives: If this is True, ``include`` directives will be followed. :raise FileNotFoundError: If *filename* does not exist. :raise InvalidConfigError: If the configuration format is invalid. Also if any of the included files do not exist. :return: A dictionary. Section names are prepended to the option names. """ filename = path.norm(filename) if not basedir: basedir = path.dirname(filename) if not path.isfile(filename): raise FileNotFoundError(filename) logger.debug('reading configuration file:', filename) parser = configparser.SafeConfigParser() try: parser.read([filename]) except configparser.Error as exc: raise InvalidConfigError('"{}": {}'.format(filename, exc)) result = {} for section in parser.sections(): match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section) if match: if not follow_include_directives: continue ifile, if_exists = match.groups() ifile = path.norm(ifile, basedir) try: result.update(read_config_file(ifile)) except FileNotFoundError as exc: if not if_exists: raise InvalidConfigError( 'file "{}" included by "{}" does not exist'.format( str(exc), filename)) continue elif section == '__global__': prefix = '' else: prefix = section + '.' for option in parser.options(section): result[prefix + option] = parser.get(section, option) return result
def project_dir(self): return path.norm(path.join(self.directory, self.manifest.project_dir))
def __call__(self, value): from craftr.core.session import session if not path.isabs(value): value = path.join(session.maindir, value) return path.norm(value)
class Session(object): """ This class manages the :class:`build.Graph` and loading of Craftr modules. .. attribute:: graph A :class:`build.Graph` instance. .. attribute:: path A list of paths that will be searched for Craftr modules. .. attribute:: module The Craftr module that is currently being executed. This is an instance of the :class:`Module` class and the same as the tip of the :attr:`modulestack`. .. attribute:: modulestack A list of modules where the last element (tip) is the module that is currently being executed. .. attribute:: modules A nested dictionary that maps from name to a dictionary of version numbers mapping to :class:`Module` objects. These are the modules that have already been loaded into the session or that have been found and cached but not yet been executed. .. attribute:: preferred_versions A nested dictionary with the same structure as :attr:`modules`. This dictionary might have been loaded from a dependency lock file and specifies the preferred version to load for a specific module, assuming that the criteria specified in the loading module's manifest is less strict. Note that Craftr will error if a preferred version can not be found. .. attribute:: maindir The main directory from which Craftr was run. Craftr will switch to the build directory at a later point, which is why we keep this member for reference. .. attribute:: builddir The absolute path to the build directory. .. attribute:: main_module The main :class:`Module`. .. attribute:: options A dictionary of options that are passed down to Craftr modules. .. attributes:: cache A JSON object that will be loaded from the current workspace's cache file and written back when Craftr exits without errors. The cache can contain anything and can be modified by everything, however it should be assured that no name conflicts and accidental modifications/deletes occur. Reserved keywords in the cache are ``"build"`` and ``"loaders"``. """ #: The current session object. Create it with :meth:`start` and destroy #: it with :meth:`end`. current = None #: Diretory that contains the Craftr standard library. stl_dir = path.norm(path.join(__file__, '../../stl')) stl_auxiliary_dir = path.norm(path.join(__file__, '../../stl_auxiliary')) def __init__(self, maindir=None): self.maindir = path.norm(maindir or path.getcwd()) self.builddir = path.join(self.maindir, 'build') self.graph = build.Graph() self.path = [ self.stl_dir, self.stl_auxiliary_dir, self.maindir, path.join(self.maindir, 'craftr/modules') ] self.modulestack = [] self.modules = {} self.preferred_versions = {} self.main_module = None self.options = {} self.cache = {} self.tasks = {} self._tempdir = None self._manifest_cache = {} # maps manifest_filename: manifest self._refresh_cache = True def __enter__(self): if Session.current: raise RuntimeError('a session was already created') Session.current = self return Session.current def __exit__(self, exc_value, exc_type, exc_tb): if Session.current is not self: raise RuntimeError('session not in context') if self._tempdir and not self.options.get( 'craftr.keep_temporary_directory'): logger.debug('removing temporary directory:', self._tempdir) try: path.remove(self._tempdir, recursive=True) except OSError as exc: logger.debug('error:', exc, indent=1) finally: self._tempdir = None Session.current = None @property def module(self): if self.modulestack: return self.modulestack[-1] return None def read_cache(self, fp): cache = json.load(fp) if not isinstance(cache, dict): raise ValueError( 'Craftr Session cache must be a JSON object, got {}'.format( type(cache).__name__)) self.cache = cache def write_cache(self, fp): json.dump(self.cache, fp, indent='\t') def expand_relative_options(self, module_name=None): """ After the main module has been detected, relative option names (starting with ``.``) should be converted to absolute option names. This is what the method does. """ if not module_name and not self.main_module: raise RuntimeError('main_module not set') if not module_name: module_name = self.main_module.manifest.name for key in tuple(self.options.keys()): if key.startswith('.'): self.options[module_name + key] = self.options.pop(key) def get_temporary_directory(self): """ Returns a writable temporary directory that is primarily used by loaders to store temporary files. The temporary directory will be deleted when the Session context ends unless the ``craftr.keep_temporary_directory`` option is set. :raise RuntimeError: If the session is not currently in context. """ if Session.current is not self: raise RuntimeError('session not in context') if not self._tempdir: self._tempdir = path.join(self.builddir, '.temp') logger.debug('created temporary directory:', self._tempdir) return self._tempdir def parse_manifest(self, filename): """ Parse a manifest by filename and add register the module to the module cache. Returns the :class:`Module` object. If the manifest has already been parsed, it will not be re-parsed. :raise Manifest.Invalid: If the manifest is invalid. :return: :const:`None` if the manifest is a duplicate of an already parsed manifest (determined by name and version), otherwise the :class:`Module` object for the manifest's module. """ filename = path.norm(path.abs(filename)) if filename in self._manifest_cache: manifest = self._manifest_cache[filename] return self.find_module(manifest.name, manifest.version) manifest = Manifest.parse(filename) self._manifest_cache[filename] = manifest versions = self.modules.setdefault(manifest.name, {}) if manifest.version in versions: other = versions[manifest.version].manifest.filename logger.debug('multiple occurences of "{}-{}" found\n' ' - {}\n - {}'.format(manifest.name, manifest.version, filename, other)) module = None else: logger.debug('parsed manifest: {}-{} ({})'.format( manifest.name, manifest.version, filename)) module = Module(path.dirname(filename), manifest) versions[manifest.version] = module return module def update_manifest_cache(self, force=False): if not self._refresh_cache and not force: return self._refresh_cache = False for directory in self.path: choices = [] choices.extend( [path.join(directory, x) for x in MANIFEST_FILENAMES]) for item in path.easy_listdir(directory): choices.extend([ path.join(directory, item, x) for x in MANIFEST_FILENAMES ]) choices.extend([ path.join(directory, item, 'craftr', x) for x in MANIFEST_FILENAMES ]) for filename in map(path.norm, choices): if filename in self._manifest_cache: continue # don't parse a manifest that we already parsed if not path.isfile(filename): continue try: self.parse_manifest(filename) except Manifest.Invalid as exc: logger.warn('invalid manifest found:', filename) logger.warn(exc, indent=1) def find_module(self, name, version, resolve_preferred_version=True): """ Finds a module in the :attr:`path` matching the specified *name* and *version*. :param name: The name of the module. :param version: A :class:`VersionCriteria`, :class:`Version` or string in a VersionCritiera format. :param resolve_preferred_version: If this parameter is True (default) and a preferred version is specified in :attr:`preferred_versions`, that preferred version is loaded or :class:`ModuleNotFound` is raised. :raise ModuleNotFound: If the module can not be found. :return: :class:`Module` """ argspec.validate('name', name, {'type': str}) argspec.validate('version', version, {'type': [str, Version, VersionCriteria]}) if name in renames.renames: logger.warn('"{}" is deprecated, use "{}" instead'.format( name, renames.renames[name])) name = renames.renames[name] if isinstance(version, str): try: version = Version(version) except ValueError as exc: version = VersionCriteria(version) if session.module and resolve_preferred_version: data = self.preferred_versions.get(session.module.manifest.name) if data is not None: versions = data.get(str(session.module.manifest.version)) if versions is not None: preferred_version = versions.get(name) if preferred_version is not None: version = Version(preferred_version) logger.debug( 'note: loading preferred version {} of module "{}" ' 'requested by module "{}"'.format( version, name, session.module.ident)) self.update_manifest_cache() if name in self.modules: if isinstance(version, Version): if version in self.modules[name]: return self.modules[name][version] raise ModuleNotFound(name, version) for module in sorted(self.modules[name].values(), key=lambda x: x.manifest.version, reverse=True): if version(module.manifest.version): return module raise ModuleNotFound(name, version)
def scriptfile(self): return path.norm(path.join(self.directory, self.manifest.main))