def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
logger.debug('multiple occurences of "{}-{}" found, '
'one of which is located at "{}"'.format(manifest.name,
manifest.version, filename))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def load_file(filename, export_default_namespace=True):
"""
Loads a Python file into a new module-like object and returns it. The
*filename* is assumed relative to the currently executed module's
directory (NOT the project directory which can be different).
"""
module = session.module
__name__ = module.ident + ':' + filename
if not path.isabs(filename):
filename = path.join(module.directory, filename)
filename = path.norm(filename)
with open(filename, 'r') as fp:
code = compile(fp.read(), filename, 'exec')
scope = Namespace()
if export_default_namespace:
vars(scope).update(module.get_init_globals())
scope.__module__ = module.namespace
scope.__file__ = filename
scope.__name__ = __name__
exec(code, vars(scope))
return scope
def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
other = versions[manifest.version].manifest.filename
logger.debug('multiple occurences of "{}-{}" found\n'
' - {}\n - {}'.format(manifest.name,
manifest.version, filename,
other))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def load_file(filename, export_default_namespace=True):
"""
Loads a Python file into a new module-like object and returns it. The
*filename* is assumed relative to the currently executed module's
directory (NOT the project directory which can be different).
"""
module = session.module
__name__ = module.ident + ':' + filename
if not path.isabs(filename):
filename = path.join(module.directory, filename)
filename = path.norm(filename)
module.dependent_files.append(filename)
with open(filename, 'r') as fp:
code = compile(fp.read(), filename, 'exec')
scope = Namespace()
if export_default_namespace:
vars(scope).update(module.get_init_globals())
scope.__module__ = module.namespace
scope.__file__ = filename
scope.__name__ = __name__
exec(code, vars(scope))
return scope
def run(self):
"""
Loads the code of the main Craftr build script as specified in the modules
manifest and executes it. Note that this must occur in a context where
the :data:`session` is available.
:raise RuntimeError: If there is no current :data:`session` or if the
module was already executed.
"""
if not session:
raise RuntimeError('no current session')
if self.executed:
raise RuntimeError('already run')
self.executed = True
self.init_options()
script_fn = path.norm(path.join(self.directory, self.manifest.main))
with open(script_fn) as fp:
code = compile(fp.read(), script_fn, 'exec')
vars(self.namespace).update(self.get_init_globals())
self.namespace.__file__ = script_fn
self.namespace.__name__ = self.manifest.name
self.namespace.__version__ = str(self.manifest.version)
try:
session.modulestack.append(self)
exec(code, vars(self.namespace))
finally:
assert session.modulestack.pop() is self
def local(rel_path): """ Given a relative path, returns the absolute path relative to the current module's project directory. """ parent = session.module.project_dir return path.norm(rel_path, parent)
def local(rel_path): """ Given a relative path, returns the absolute path relative to the current module's project directory. """ parent = session.module.namespace.project_dir return path.norm(rel_path, parent)
def __init__(self, maindir=None):
self.maindir = path.norm(maindir or path.getcwd())
self.builddir = path.join(self.maindir, 'build')
self.graph = build.Graph()
self.path = [self.stl_dir, self.maindir, path.join(self.maindir, 'craftr/modules')]
self.modulestack = []
self.modules = {}
self.options = {}
self.cache = {'loaders': {}}
self._tempdir = None
self._manifest_cache = {} # maps manifest_filename: manifest
self._refresh_cache = True
def __lshift__(self, other):
"""
Adds *other* as an implicit dependency to the target.
:param other: A :class:`Target` or :class:`str`.
:return: ``self``
"""
if isinstance(other, Target):
self.implicit_deps += other.outputs
elif isinstance(other, str):
self.implicit_deps.append(path.norm(other))
else:
raise TypeError("Target.__lshift__() expected Target or str")
return self
def __lshift__(self, other):
"""
Adds *other* as an implicit dependency to the target.
:param other: A :class:`Target` or :class:`str`.
:return: ``self``
"""
if isinstance(other, Target):
self.implicit_deps += other.outputs
elif isinstance(other, str):
self.implicit_deps.append(path.norm(other))
else:
raise TypeError("Target.__lshift__() expected Target or str")
return self
def __init__(self, maindir=None):
self.maindir = path.norm(maindir or path.getcwd())
self.builddir = path.join(self.maindir, 'build')
self.graph = build.Graph()
self.path = [
self.stl_dir, self.stl_auxiliary_dir, self.maindir,
path.join(self.maindir, 'craftr/modules')
]
self.modulestack = []
self.modules = {}
self.preferred_versions = {}
self.main_module = None
self.options = {}
self.cache = {}
self.tasks = {}
self._tempdir = None
self._manifest_cache = {} # maps manifest_filename: manifest
self._refresh_cache = True
def execute(self, parser, args):
if hasattr(args, 'include_path'):
session.path.extend(map(path.norm, args.include_path))
# Help-command preprocessing. Check if we're to show the help on a builtin
# object, otherwise extract the module name if applicable.
if self.mode == 'help':
if not args.name:
help('craftr')
return 0
if args.name in vars(craftr.defaults):
help(getattr(craftr.defaults, args.name))
return 0
# Check if we have an absolute symbol reference.
if ':' in args.name:
if args.module:
parser.error(
'-m/--module option conflicting with name argument: "{}"'
.format(args.name))
args.module, args.name = args.name.split(':', 1)
module = self._find_module(parser, args)
session.main_module = module
self.ninja_bin, self.ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(path.norm(args.build_dir, INIT_DIR))
path.makedirs(session.builddir)
os.chdir(session.builddir)
self.cachefile = path.join(session.builddir, '.craftrcache')
# Prepare options, loaders and execute.
if self.mode in ('export', 'run', 'help'):
return self._export_run_or_help(args, module)
elif self.mode == 'dump-options':
return self._dump_options(args, module)
elif self.mode == 'dump-deptree':
return self._dump_deptree(args, module)
elif self.mode in ('build', 'clean'):
return self._build_or_clean(args)
elif self.mode == 'lock':
self._create_lockfile()
else:
raise RuntimeError("mode: {}".format(self.mode))
def run(self):
"""
Loads the code of the main Craftr build script as specified in the modules
manifest and executes it. Note that this must occur in a context where
the :data:`session` is available.
:raise RuntimeError: If there is no current :data:`session` or if the
module was already executed.
"""
if not session:
raise RuntimeError('no current session')
if self.executed:
raise RuntimeError('already run')
self.executed = True
self.init_options()
self.init_loader()
script_fn = path.norm(path.join(self.directory, self.manifest.main))
with open(script_fn) as fp:
code = compile(fp.read(), script_fn, 'exec')
from craftr import defaults
for key, value in vars(defaults).items():
if not key.startswith('_'):
vars(self.namespace)[key] = value
vars(self.namespace).update({
'__file__': script_fn,
'__name__': self.manifest.name,
'__version__': str(self.manifest.version),
'options': self.options,
'loader': self.loader,
'project_dir': self.project_dir,
})
try:
session.modulestack.append(self)
exec(code, vars(self.namespace))
finally:
assert session.modulestack.pop() is self
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug("reading configuration file:", filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename))
continue
elif section == "__global__":
prefix = ""
else:
prefix = section + "."
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug('reading configuration file:', filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError(
'file "{}" included by "{}" does not exist'.format(
str(exc), filename))
continue
elif section == '__global__':
prefix = ''
else:
prefix = section + '.'
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
def project_dir(self): return path.norm(path.join(self.directory, self.manifest.project_dir))
def __call__(self, value):
from craftr.core.session import session
if not path.isabs(value):
value = path.join(session.maindir, value)
return path.norm(value)
def __call__(self, value):
from craftr.core.session import session
if not path.isabs(value):
value = path.join(session.maindir, value)
return path.norm(value)
class Session(object):
"""
This class manages the :class:`build.Graph` and loading of Craftr modules.
.. attribute:: graph
A :class:`build.Graph` instance.
.. attribute:: path
A list of paths that will be searched for Craftr modules.
.. attribute:: module
The Craftr module that is currently being executed. This is an instance
of the :class:`Module` class and the same as the tip of the
:attr:`modulestack`.
.. attribute:: modulestack
A list of modules where the last element (tip) is the module that is
currently being executed.
.. attribute:: modules
A nested dictionary that maps from name to a dictionary of version
numbers mapping to :class:`Module` objects. These are the modules that
have already been loaded into the session or that have been found and
cached but not yet been executed.
.. attribute:: preferred_versions
A nested dictionary with the same structure as :attr:`modules`. This
dictionary might have been loaded from a dependency lock file and specifies
the preferred version to load for a specific module, assuming that the
criteria specified in the loading module's manifest is less strict. Note
that Craftr will error if a preferred version can not be found.
.. attribute:: maindir
The main directory from which Craftr was run. Craftr will switch to the
build directory at a later point, which is why we keep this member for
reference.
.. attribute:: builddir
The absolute path to the build directory.
.. attribute:: main_module
The main :class:`Module`.
.. attribute:: options
A dictionary of options that are passed down to Craftr modules.
.. attributes:: cache
A JSON object that will be loaded from the current workspace's cache
file and written back when Craftr exits without errors. The cache can
contain anything and can be modified by everything, however it should
be assured that no name conflicts and accidental modifications/deletes
occur.
Reserved keywords in the cache are ``"build"`` and ``"loaders"``.
"""
#: The current session object. Create it with :meth:`start` and destroy
#: it with :meth:`end`.
current = None
#: Diretory that contains the Craftr standard library.
stl_dir = path.norm(path.join(__file__, '../../stl'))
stl_auxiliary_dir = path.norm(path.join(__file__, '../../stl_auxiliary'))
def __init__(self, maindir=None):
self.maindir = path.norm(maindir or path.getcwd())
self.builddir = path.join(self.maindir, 'build')
self.graph = build.Graph()
self.path = [
self.stl_dir, self.stl_auxiliary_dir, self.maindir,
path.join(self.maindir, 'craftr/modules')
]
self.modulestack = []
self.modules = {}
self.preferred_versions = {}
self.main_module = None
self.options = {}
self.cache = {}
self.tasks = {}
self._tempdir = None
self._manifest_cache = {} # maps manifest_filename: manifest
self._refresh_cache = True
def __enter__(self):
if Session.current:
raise RuntimeError('a session was already created')
Session.current = self
return Session.current
def __exit__(self, exc_value, exc_type, exc_tb):
if Session.current is not self:
raise RuntimeError('session not in context')
if self._tempdir and not self.options.get(
'craftr.keep_temporary_directory'):
logger.debug('removing temporary directory:', self._tempdir)
try:
path.remove(self._tempdir, recursive=True)
except OSError as exc:
logger.debug('error:', exc, indent=1)
finally:
self._tempdir = None
Session.current = None
@property
def module(self):
if self.modulestack:
return self.modulestack[-1]
return None
def read_cache(self, fp):
cache = json.load(fp)
if not isinstance(cache, dict):
raise ValueError(
'Craftr Session cache must be a JSON object, got {}'.format(
type(cache).__name__))
self.cache = cache
def write_cache(self, fp):
json.dump(self.cache, fp, indent='\t')
def expand_relative_options(self, module_name=None):
"""
After the main module has been detected, relative option names (starting
with ``.``) should be converted to absolute option names. This is what
the method does.
"""
if not module_name and not self.main_module:
raise RuntimeError('main_module not set')
if not module_name:
module_name = self.main_module.manifest.name
for key in tuple(self.options.keys()):
if key.startswith('.'):
self.options[module_name + key] = self.options.pop(key)
def get_temporary_directory(self):
"""
Returns a writable temporary directory that is primarily used by loaders
to store temporary files. The temporary directory will be deleted when
the Session context ends unless the ``craftr.keep_temporary_directory``
option is set.
:raise RuntimeError: If the session is not currently in context.
"""
if Session.current is not self:
raise RuntimeError('session not in context')
if not self._tempdir:
self._tempdir = path.join(self.builddir, '.temp')
logger.debug('created temporary directory:', self._tempdir)
return self._tempdir
def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
other = versions[manifest.version].manifest.filename
logger.debug('multiple occurences of "{}-{}" found\n'
' - {}\n - {}'.format(manifest.name,
manifest.version, filename,
other))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def update_manifest_cache(self, force=False):
if not self._refresh_cache and not force:
return
self._refresh_cache = False
for directory in self.path:
choices = []
choices.extend(
[path.join(directory, x) for x in MANIFEST_FILENAMES])
for item in path.easy_listdir(directory):
choices.extend([
path.join(directory, item, x) for x in MANIFEST_FILENAMES
])
choices.extend([
path.join(directory, item, 'craftr', x)
for x in MANIFEST_FILENAMES
])
for filename in map(path.norm, choices):
if filename in self._manifest_cache:
continue # don't parse a manifest that we already parsed
if not path.isfile(filename):
continue
try:
self.parse_manifest(filename)
except Manifest.Invalid as exc:
logger.warn('invalid manifest found:', filename)
logger.warn(exc, indent=1)
def find_module(self, name, version, resolve_preferred_version=True):
"""
Finds a module in the :attr:`path` matching the specified *name* and
*version*.
:param name: The name of the module.
:param version: A :class:`VersionCriteria`, :class:`Version` or string
in a VersionCritiera format.
:param resolve_preferred_version: If this parameter is True (default)
and a preferred version is specified in :attr:`preferred_versions`,
that preferred version is loaded or :class:`ModuleNotFound` is raised.
:raise ModuleNotFound: If the module can not be found.
:return: :class:`Module`
"""
argspec.validate('name', name, {'type': str})
argspec.validate('version', version,
{'type': [str, Version, VersionCriteria]})
if name in renames.renames:
logger.warn('"{}" is deprecated, use "{}" instead'.format(
name, renames.renames[name]))
name = renames.renames[name]
if isinstance(version, str):
try:
version = Version(version)
except ValueError as exc:
version = VersionCriteria(version)
if session.module and resolve_preferred_version:
data = self.preferred_versions.get(session.module.manifest.name)
if data is not None:
versions = data.get(str(session.module.manifest.version))
if versions is not None:
preferred_version = versions.get(name)
if preferred_version is not None:
version = Version(preferred_version)
logger.debug(
'note: loading preferred version {} of module "{}" '
'requested by module "{}"'.format(
version, name, session.module.ident))
self.update_manifest_cache()
if name in self.modules:
if isinstance(version, Version):
if version in self.modules[name]:
return self.modules[name][version]
raise ModuleNotFound(name, version)
for module in sorted(self.modules[name].values(),
key=lambda x: x.manifest.version,
reverse=True):
if version(module.manifest.version):
return module
raise ModuleNotFound(name, version)
def scriptfile(self):
return path.norm(path.join(self.directory, self.manifest.main))
def project_dir(self):
return path.norm(path.join(self.directory, self.manifest.project_dir))
