def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
logger.debug('multiple occurences of "{}-{}" found, '
'one of which is located at "{}"'.format(manifest.name,
manifest.version, filename))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
other = versions[manifest.version].manifest.filename
logger.debug('multiple occurences of "{}-{}" found\n'
' - {}\n - {}'.format(manifest.name,
manifest.version, filename,
other))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def get_ninja_info():
# Make sure the Ninja executable exists and find its version.
ninja_bin = session.options.get('global.ninja') or \
session.options.get('craftr.ninja') or os.getenv('NINJA', 'ninja')
ninja_bin = shell.find_program(ninja_bin)
ninja_version = get_ninja_version(ninja_bin)
logger.debug('Ninja executable:', ninja_bin)
logger.debug('Ninja version:', ninja_version)
return ninja_bin, ninja_version
def get_ninja_info():
# Make sure the Ninja executable exists and find its version.
ninja_bin = session.options.get('global.ninja') or \
session.options.get('craftr.ninja') or os.getenv('NINJA', 'ninja')
ninja_bin = shell.find_program(ninja_bin)
ninja_version = get_ninja_version(ninja_bin)
logger.debug('Ninja executable:', ninja_bin)
logger.debug('Ninja version:', ninja_version)
return ninja_bin, ninja_version
def find_module(self, name, version, resolve_preferred_version=True):
"""
Finds a module in the :attr:`path` matching the specified *name* and
*version*.
:param name: The name of the module.
:param version: A :class:`VersionCriteria`, :class:`Version` or string
in a VersionCritiera format.
:param resolve_preferred_version: If this parameter is True (default)
and a preferred version is specified in :attr:`preferred_versions`,
that preferred version is loaded or :class:`ModuleNotFound` is raised.
:raise ModuleNotFound: If the module can not be found.
:return: :class:`Module`
"""
argspec.validate('name', name, {'type': str})
argspec.validate('version', version,
{'type': [str, Version, VersionCriteria]})
if name in renames.renames:
logger.warn('"{}" is deprecated, use "{}" instead'.format(
name, renames.renames[name]))
name = renames.renames[name]
if isinstance(version, str):
try:
version = Version(version)
except ValueError as exc:
version = VersionCriteria(version)
if session.module and resolve_preferred_version:
data = self.preferred_versions.get(session.module.manifest.name)
if data is not None:
versions = data.get(str(session.module.manifest.version))
if versions is not None:
preferred_version = versions.get(name)
if preferred_version is not None:
version = Version(preferred_version)
logger.debug(
'note: loading preferred version {} of module "{}" '
'requested by module "{}"'.format(
version, name, session.module.ident))
self.update_manifest_cache()
if name in self.modules:
if isinstance(version, Version):
if version in self.modules[name]:
return self.modules[name][version]
raise ModuleNotFound(name, version)
for module in sorted(self.modules[name].values(),
key=lambda x: x.manifest.version,
reverse=True):
if version(module.manifest.version):
return module
raise ModuleNotFound(name, version)
def get_ninja_info():
# Make sure the Ninja executable exists and find its version.
ninja_bin = (
session.options.get("global.ninja") or session.options.get("craftr.ninja") or os.getenv("NINJA", "ninja")
)
ninja_bin = shell.find_program(ninja_bin)
ninja_version = get_ninja_version(ninja_bin)
logger.debug("Ninja executable:", ninja_bin)
logger.debug("Ninja version:", ninja_version)
return ninja_bin, ninja_version
def write_cache(cachefile):
# Write back the cache.
try:
path.makedirs(path.dirname(cachefile))
with open(cachefile, 'w') as fp:
session.write_cache(fp)
except OSError as exc:
logger.error('error writing cache file:', cachefile)
logger.error(exc, indent=1)
else:
logger.debug('cache written:', cachefile)
def write_cache(cachefile):
# Write back the cache.
try:
path.makedirs(path.dirname(cachefile))
with open(cachefile, 'w') as fp:
session.write_cache(fp)
except OSError as exc:
logger.error('error writing cache file:', cachefile)
logger.error(exc, indent=1)
else:
logger.debug('cache written:', cachefile)
def __exit__(self, exc_value, exc_type, exc_tb):
if Session.current is not self:
raise RuntimeError('session not in context')
if self._tempdir and not self.options.get('craftr.keep_temporary_directory'):
logger.debug('removing temporary directory:', self._tempdir)
try:
path.remove(self._tempdir, recursive=True)
except OSError as exc:
logger.debug('error:', exc, indent=1)
finally:
self._tempdir = None
Session.current = None
def __cleanup(self, parser, args):
"""
Switch back to the original directory and check if we can clean up
the build directory.
"""
os.chdir(session.maindir)
if os.path.isdir(
session.builddir) and not os.listdir(session.builddir):
logger.debug('note: cleanup empty build directory:',
session.builddir)
os.rmdir(session.builddir)
def __exit__(self, exc_value, exc_type, exc_tb):
if Session.current is not self:
raise RuntimeError('session not in context')
if self._tempdir and not self.options.get(
'craftr.keep_temporary_directory'):
logger.debug('removing temporary directory:', self._tempdir)
try:
path.remove(self._tempdir, recursive=True)
except OSError as exc:
logger.debug('error:', exc, indent=1)
finally:
self._tempdir = None
Session.current = None
def get_temporary_directory(self):
"""
Returns a writable temporary directory that is primarily used by loaders
to store temporary files. The temporary directory will be deleted when
the Session context ends unless the ``craftr.keep_temporary_directory``
option is set.
:raise RuntimeError: If the session is not currently in context.
"""
if Session.current is not self:
raise RuntimeError('session not in context')
if not self._tempdir:
self._tempdir = path.join(self.builddir, '.temp')
logger.debug('created temporary directory:', self._tempdir)
return self._tempdir
def get_temporary_directory(self):
"""
Returns a writable temporary directory that is primarily used by loaders
to store temporary files. The temporary directory will be deleted when
the Session context ends unless the ``craftr.keep_temporary_directory``
option is set.
:raise RuntimeError: If the session is not currently in context.
"""
if Session.current is not self:
raise RuntimeError('session not in context')
if not self._tempdir:
self._tempdir = path.join(self.builddir, '.temp')
logger.debug('created temporary directory:', self._tempdir)
return self._tempdir
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, 'craftr')
path.makedirs(directory)
mfile = path.join(directory, 'manifest.' + args.format)
sfile = path.join(directory, 'Craftrfile')
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, 'w') as fp:
if args.format == 'cson':
lines = textwrap.dedent('''
name: "%s"
version: "%s"
project_dir: ".."
author: ""
url: ""
dependencies: {}
options: {}
''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[2]
elif args.format == 'json':
lines = textwrap.dedent('''
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[3]
fp.write('\n'.join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, 'w') as fp:
print('# {}'.format(args.name), file=fp)
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, "craftr")
path.makedirs(directory)
mfile = path.join(directory, MANIFEST_FILENAME)
sfile = path.join(directory, "Craftrfile")
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, "w") as fp:
lines = (
textwrap.dedent(
"""
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}\n"""
% (args.name, args.version)
)
.lstrip()
.split("\n")
)
if not args.nested:
del lines[3]
fp.write("\n".join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, "w") as fp:
print("# {}".format(args.name), file=fp)
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug('reading configuration file:', filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError(
'file "{}" included by "{}" does not exist'.format(
str(exc), filename))
continue
elif section == '__global__':
prefix = ''
else:
prefix = section + '.'
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
def _build_or_clean(self, args):
"""
Will be called for the 'build' and 'clean' modes. Loads the Craftr
cache and invokes Ninja.
"""
# Read the cache and parse command-line options.
if not read_cache(True):
sys.exit(1)
parse_cmdline_options(session.cache['build']['options'])
main = session.cache['build']['main']
available_targets = frozenset(session.cache['build']['targets'])
available_modules = unserialise_loaded_module_info(
session.cache['build']['modules'])
logger.debug('build main module:', main)
session.expand_relative_options(get_volatile_module_version(main)[0])
# Check if any of the modules changed, so we can let the user know he
# might have to re-export the build files.
changed_modules = []
for name, versions in available_modules.items():
for version, info in versions.items():
if info['changed']:
changed_modules.append('{}-{}'.format(name, version))
if changed_modules:
if len(changed_modules) == 1:
logger.info(
'note: module "{}" has changed, maybe you should re-export'
.format(changed_modules[0]))
else:
logger.info(
'note: some modules have changed, maybe you should re-export'
)
for name in changed_modules:
logger.info(' -', name)
# Check the targets and if they exist.
targets = []
for target_name in args.targets:
if '.' not in target_name:
target_name = main + '.' + target_name
elif target_name.startswith('.'):
target_name = main + target_name
module_name, target_name = target_name.rpartition('.')[::2]
module_name, version = get_volatile_module_version(module_name)
if module_name not in available_modules:
error('no such module:', module_name)
if not version:
version = max(available_modules[module_name].keys())
target_name = craftr.targetbuilder.get_full_name(
target_name, module_name=module_name, version=version)
if target_name not in available_targets:
logger.error('no such target: {}'.format(target_name))
return 1
targets.append(target_name)
# Make sure we get all the output before running the subcommand.
logger.flush()
# Execute the ninja build.
cmd = [self.ninja_bin]
if args.verbose:
cmd += ['-v']
if self.mode == 'clean':
cmd += ['-t', 'clean']
if not args.recursive:
cmd += ['-r']
cmd += targets
return shell.run(cmd).returncode
def _export_run_or_help(self, args, module):
"""
Called when the mode is 'export' or 'run'. Will execute the specified
*module* and eventually export a Ninja manifest and Cache.
"""
read_cache(False)
session.expand_relative_options()
session.cache['build'] = {}
# Load the dependency lock information if it exists.
deplock_fn = path.join(path.dirname(module.manifest.filename),
'.dependency-lock')
if os.path.isfile(deplock_fn):
with open(deplock_fn) as fp:
session.preferred_versions = cson.load(fp)
logger.debug('note: dependency lock file "{}" loaded'.format(
deplock_fn))
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error('error:', exc)
return 1
finally:
if sys.exc_info() and self.mode == 'export':
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(self.cachefile)
# Fill the cache.
session.cache['build']['targets'] = list(session.graph.targets.keys())
session.cache['build']['modules'] = serialise_loaded_module_info()
session.cache['build']['main'] = module.ident
session.cache['build']['options'] = args.options
session.cache['build']['dependency_lock_filename'] = deplock_fn
if self.mode == 'export':
# Add the Craftr_run_command variable which is necessary for tasks
# to properly executed.
run_command = ['craftr', '-q', '-P', path.rel(session.maindir)]
if args.no_config: run_command += ['-C']
run_command += ['-c' + x for x in args.config]
run_command += ['run']
if args.module: run_command += ['-m', args.module]
run_command += ['-i' + x for x in args.include_path]
run_command += ['-b', path.rel(session.builddir)]
session.graph.vars['Craftr_run_command'] = shell.join(run_command)
write_cache(self.cachefile)
# Write the Ninja manifest.
with open("build.ninja", 'w') as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(self.ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
logger.info('exported "build.ninja"')
return 0
elif self.mode == 'run':
if args.task:
if args.task not in session.graph.tasks:
logger.error('no such task exists: "{}"'.format(args.task))
return 1
task = session.graph.tasks[args.task]
return task.invoke(args.task_args)
return 0
elif self.mode == 'help':
if args.name not in vars(module.namespace):
logger.error('symbol not found: "{}:{}"'.format(
module.manifest.name, args.name))
return 1
help(getattr(module.namespace, args.name))
return 0
assert False, "unhandled mode: {}".format(self.mode)
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.mode == "export":
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error("{} (note: you have to escape > and < characters)".format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error("module not found: " + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, ".craftrcache")
if not read_cache(cachefile) and self.mode != "export":
logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode))
logger.error("Make sure to generate a build tree with 'craftr export'")
return 1
# Prepare options, loaders and execute.
if self.mode == "export":
session.expand_relative_options(module.manifest.name)
session.cache["build"] = {}
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error("error:", exc)
return 1
finally:
if sys.exc_info():
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(cachefile)
# Write the cache back.
session.cache["build"]["targets"] = list(session.graph.targets.keys())
session.cache["build"]["main"] = module.ident
session.cache["build"]["options"] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", "w") as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache["build"]["options"])
main = session.cache["build"]["main"]
available_targets = frozenset(session.cache["build"]["targets"])
logger.debug("build main module:", main)
session.expand_relative_options(get_volatile_module_version(main)[0])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if "." not in target:
target = main + "." + target
elif target.startswith("."):
target = main + target
module_name, target = target.rpartition(".")[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or "*")
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error("no such target: {}".format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ["-v"]
if self.mode == "clean":
cmd += ["-t", "clean"]
if not args.recursive:
cmd += ["-r"]
cmd += targets
return shell.run(cmd).returncode
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug("reading configuration file:", filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename))
continue
elif section == "__global__":
prefix = ""
else:
prefix = section + "."
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
def load(self, context, cache):
if cache is not None and path.isdir(cache.get("directory", "")):
# Check if the requested version changes.
url_template = context.expand_variables(cache.get("url_template", ""))
if url_template == cache.get("url"):
self.directory = cache["directory"]
logger.info("Reusing cached directory: {}".format(path.rel(self.directory, nopar=True)))
return cache
else:
logger.info("Cached URL is outdated:", cache.get("url"))
directory = None
archive = None
delete_after_extract = True
for url_template in self.urls:
url = context.expand_variables(url_template)
if not url:
continue
if url.startswith("file://"):
name = url[7:]
if path.isdir(name):
logger.info("Using directory", url)
directory = name
break
elif path.isfile(name):
logger.info("Using archive", url)
archive = name
delete_after_extract = False
break
error = None
else:
error = None
try:
progress = lambda d: self._download_progress(url, context, d)
archive, reused = httputils.download_file(
url, directory=context.get_temporary_directory(), on_exists="skip", progress=progress
)
except (httputils.URLError, httputils.HTTPError) as exc:
error = exc
except self.DownloadAlreadyExists as exc:
directory = exc.directory
logger.info("Reusing existing directory", directory)
else:
if reused:
logger.info("Reusing cached download", path.basename(archive))
break
if error:
logger.info("Error reading", url, ":", error)
if directory or archive:
logger.debug("URL applies: {}".format(url))
if not directory and archive:
suffix, directory = self._get_archive_unpack_info(context, archive)
logger.info(
'Unpacking "{}" to "{}" ...'.format(path.rel(archive, nopar=True), path.rel(directory, nopar=True))
)
nr.misc.archive.extract(
archive,
directory,
suffix=suffix,
unpack_single_dir=True,
check_extract_file=self._check_extract_file,
progress_callback=self._extract_progress,
)
elif not directory:
raise LoaderError(self, "no URL matched")
self.directory = directory
with open(path.join(self.directory, ".craftr_downloadurl"), "w") as fp:
fp.write(url)
return {"directory": directory, "url_template": url_template, "url": url}