def update_manifest_cache(self, force=False):
if not self._refresh_cache and not force:
return
self._refresh_cache = False
for directory in self.path:
choices = []
choices.extend(
[path.join(directory, x) for x in MANIFEST_FILENAMES])
for item in path.easy_listdir(directory):
choices.extend([
path.join(directory, item, x) for x in MANIFEST_FILENAMES
])
choices.extend([
path.join(directory, item, 'craftr', x)
for x in MANIFEST_FILENAMES
])
for filename in map(path.norm, choices):
if filename in self._manifest_cache:
continue # don't parse a manifest that we already parsed
if not path.isfile(filename):
continue
try:
self.parse_manifest(filename)
except Manifest.Invalid as exc:
logger.warn('invalid manifest found:', filename)
logger.warn(exc, indent=1)
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, 'craftr')
path.makedirs(directory)
mfile = path.join(directory, 'manifest.' + args.format)
sfile = path.join(directory, 'Craftrfile')
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, 'w') as fp:
if args.format == 'cson':
lines = textwrap.dedent('''
name: "%s"
version: "%s"
project_dir: ".."
author: ""
url: ""
dependencies: {}
options: {}
''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[2]
elif args.format == 'json':
lines = textwrap.dedent('''
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[3]
fp.write('\n'.join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, 'w') as fp:
print('# {}'.format(args.name), file=fp)
def __init__(self, maindir=None):
self.maindir = path.norm(maindir or path.getcwd())
self.builddir = path.join(self.maindir, 'build')
self.graph = build.Graph()
self.path = [self.stl_dir, self.maindir, path.join(self.maindir, 'craftr/modules')]
self.modulestack = []
self.modules = {}
self.options = {}
self.cache = {'loaders': {}}
self._tempdir = None
self._manifest_cache = {} # maps manifest_filename: manifest
self._refresh_cache = True
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, "craftr")
path.makedirs(directory)
mfile = path.join(directory, MANIFEST_FILENAME)
sfile = path.join(directory, "Craftrfile")
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, "w") as fp:
lines = (
textwrap.dedent(
"""
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}\n"""
% (args.name, args.version)
)
.lstrip()
.split("\n")
)
if not args.nested:
del lines[3]
fp.write("\n".join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, "w") as fp:
print("# {}".format(args.name), file=fp)
def load_file(filename, export_default_namespace=True):
"""
Loads a Python file into a new module-like object and returns it. The
*filename* is assumed relative to the currently executed module's
directory (NOT the project directory which can be different).
"""
module = session.module
__name__ = module.ident + ':' + filename
if not path.isabs(filename):
filename = path.join(module.directory, filename)
filename = path.norm(filename)
with open(filename, 'r') as fp:
code = compile(fp.read(), filename, 'exec')
scope = Namespace()
if export_default_namespace:
vars(scope).update(module.get_init_globals())
scope.__module__ = module.namespace
scope.__file__ = filename
scope.__name__ = __name__
exec(code, vars(scope))
return scope
def run(self):
"""
Loads the code of the main Craftr build script as specified in the modules
manifest and executes it. Note that this must occur in a context where
the :data:`session` is available.
:raise RuntimeError: If there is no current :data:`session` or if the
module was already executed.
"""
if not session:
raise RuntimeError('no current session')
if self.executed:
raise RuntimeError('already run')
self.executed = True
self.init_options()
script_fn = path.norm(path.join(self.directory, self.manifest.main))
with open(script_fn) as fp:
code = compile(fp.read(), script_fn, 'exec')
vars(self.namespace).update(self.get_init_globals())
self.namespace.__file__ = script_fn
self.namespace.__name__ = self.manifest.name
self.namespace.__version__ = str(self.manifest.version)
try:
session.modulestack.append(self)
exec(code, vars(self.namespace))
finally:
assert session.modulestack.pop() is self
def load_file(filename, export_default_namespace=True):
"""
Loads a Python file into a new module-like object and returns it. The
*filename* is assumed relative to the currently executed module's
directory (NOT the project directory which can be different).
"""
module = session.module
__name__ = module.ident + ':' + filename
if not path.isabs(filename):
filename = path.join(module.directory, filename)
filename = path.norm(filename)
module.dependent_files.append(filename)
with open(filename, 'r') as fp:
code = compile(fp.read(), filename, 'exec')
scope = Namespace()
if export_default_namespace:
vars(scope).update(module.get_init_globals())
scope.__module__ = module.namespace
scope.__file__ = filename
scope.__name__ = __name__
exec(code, vars(scope))
return scope
def export(self, writer, context, platform):
"""
Export the target to a Ninja manifest.
"""
writer.comment("target: {}".format(self.name))
writer.comment("--------" + "-" * len(self.name))
commands = platform.prepare_commands(
[list(map(str, c)) for c in self.commands])
# Check if we need to export a command file or can export the command
# directly.
if not self.environ and len(commands) == 1:
commands = [platform.prepare_single_command(commands[0], self.cwd)]
else:
filename = path.join('.commands', self.name)
command, __ = platform.write_command_file(filename,
commands,
self.inputs,
self.outputs,
cwd=self.cwd,
environ=self.environ,
foreach=self.foreach)
commands = [command]
assert len(commands) == 1
command = shell.join(commands[0], for_ninja=True)
writer.rule(self.name,
command,
pool=self.pool,
deps=self.deps,
depfile=self.depfile,
description=self.description)
if self.msvc_deps_prefix:
# We can not write msvc_deps_prefix on the rule level with Ninja
# versions older than 1.7.1. Write it global instead, but that *could*
# lead to issues...
indent = 1 if context.ninja_version > '1.7.1' else 0
writer.variable('msvc_deps_prefix', self.msvc_deps_prefix, indent)
writer.newline()
if self.foreach:
assert len(self.inputs) == len(self.outputs)
for infile, outfile in zip(self.inputs, self.outputs):
writer.build([outfile],
self.name, [infile],
implicit=self.implicit_deps,
order_only=self.order_only_deps)
else:
writer.build(self.outputs or [self.name],
self.name,
self.inputs,
implicit=self.implicit_deps,
order_only=self.order_only_deps)
if self.outputs and self.name not in self.outputs and not self.explicit:
writer.build(self.name, 'phony', self.outputs)
def __init__(self, maindir=None):
self.maindir = path.norm(maindir or path.getcwd())
self.builddir = path.join(self.maindir, 'build')
self.graph = build.Graph()
self.path = [
self.stl_dir, self.stl_auxiliary_dir, self.maindir,
path.join(self.maindir, 'craftr/modules')
]
self.modulestack = []
self.modules = {}
self.preferred_versions = {}
self.main_module = None
self.options = {}
self.cache = {}
self.tasks = {}
self._tempdir = None
self._manifest_cache = {} # maps manifest_filename: manifest
self._refresh_cache = True
def buildlocal(rel_path):
"""
Given a relative path, returns the path (still relative) to the build
directory for the current module. This is basically a shorthand for
prepending the module name and version to *path*.
"""
if path.isabs(rel_path):
return rel_path
return path.canonical(path.join(session.module.ident, rel_path))
def buildlocal(rel_path):
"""
Given a relative path, returns the path (still relative) to the build
directory for the current module. This is basically a shorthand for
prepending the module name and version to *path*.
"""
if path.isabs(rel_path):
return rel_path
return path.canonical(path.join(session.module.ident, rel_path))
def init_loader(self, recursive=False, _break_recursion=None):
"""
Check all available loaders as defined in the :attr:`manifest` until the
first loads successfully.
:param recursive: Initialize the loaders of all dependencies as well.
:raise RuntimeError: If there is no current session context.
:raise LoaderInitializationError: If none of the loaders matched.
"""
if not session:
raise RuntimeError('no current session')
if not self.manifest.loaders:
return
if _break_recursion is self:
return
if recursive:
for name, version in self.manifest.dependencies.items():
module = session.find_module(name, version)
module.init_loader(True, _break_recursion=self)
self.init_options()
if self.loader is not None:
return
logger.info('running loaders for {}'.format(self.ident))
with logger.indent():
# Read the cached loader data and create the context.
installdir = path.join(session.builddir, self.ident, 'src')
cache = session.cache['loaders'].get(self.ident)
context = LoaderContext(self.directory, self.manifest, self.options,
installdir = installdir)
context.get_temporary_directory = session.get_temporary_directory
# Check all loaders in-order.
errors = []
for loader in self.manifest.loaders:
logger.info('[+]', loader.name)
with logger.indent():
try:
if cache and loader.name == cache['name']:
new_data = loader.load(context, cache['data'])
else:
new_data = loader.load(context, None)
except manifest.LoaderError as exc:
errors.append(exc)
else:
self.loader = loader
session.cache['loaders'][self.ident] = {
'name': loader.name, 'data': new_data}
break
else:
raise LoaderInitializationError(self, errors)
def export(self, writer, context, platform):
name = str(self)[1:]
if not self.preamble and not self.environ:
self.exported_command = shlex.join(self.command)
else:
filename = path.join('.tools', name)
command, filename = platform.write_command_file(
filename, list(self.preamble) + [self.command], environ=self.environ,
accept_additional_args=True)
self.exported_command = shell.join(command)
writer.variable(name, self.exported_command)
def export(self, writer, context, platform):
"""
Export the target to a Ninja manifest.
"""
writer.comment("target: {}".format(self.name))
writer.comment("--------" + "-" * len(self.name))
commands = platform.prepare_commands([list(map(str, c)) for c in self.commands])
# Check if we need to export a command file or can export the command
# directly.
if not self.environ and len(commands) == 1:
commands = [platform.prepare_single_command(commands[0], self.cwd)]
else:
filename = path.join('.commands', self.name)
command, __ = platform.write_command_file(filename, commands,
self.inputs, self.outputs, cwd=self.cwd, environ=self.environ,
foreach=self.foreach)
commands = [command]
assert len(commands) == 1
command = shell.join(commands[0], for_ninja=True)
writer.rule(self.name, command, pool=self.pool, deps=self.deps,
depfile=self.depfile, description=self.description)
if self.msvc_deps_prefix:
# We can not write msvc_deps_prefix on the rule level with Ninja
# versions older than 1.7.1. Write it global instead, but that *could*
# lead to issues...
indent = 1 if context.ninja_version > '1.7.1' else 0
writer.variable('msvc_deps_prefix', self.msvc_deps_prefix, indent)
writer.newline()
if self.foreach:
assert len(self.inputs) == len(self.outputs)
for infile, outfile in zip(self.inputs, self.outputs):
writer.build(
[outfile],
self.name,
[infile],
implicit=self.implicit_deps,
order_only=self.order_only_deps)
else:
writer.build(
self.outputs or [self.name],
self.name,
self.inputs,
implicit=self.implicit_deps,
order_only=self.order_only_deps)
if self.outputs and self.name not in self.outputs and not self.explicit:
writer.build(self.name, 'phony', self.outputs)
def update_manifest_cache(self, force=False):
if not self._refresh_cache and not force:
return
self._refresh_cache = False
for directory in self.path:
choices = []
choices.append(path.join(directory, MANIFEST_FILENAME))
for item in path.easy_listdir(directory):
choices.append(path.join(directory, item, MANIFEST_FILENAME))
choices.append(path.join(directory, item, 'craftr', MANIFEST_FILENAME))
for filename in map(path.norm, choices):
if filename in self._manifest_cache:
continue # don't parse a manifest that we already parsed
if not path.isfile(filename):
continue
try:
self.parse_manifest(filename)
except Manifest.Invalid as exc:
logger.warn('invalid manifest found:', filename)
logger.warn(exc, indent=1)
def export(self, writer, context, platform):
name = str(self)[1:]
if not self.preamble and not self.environ:
self.exported_command = shell.join(self.command)
else:
filename = path.join('.tools', name)
command, filename = platform.write_command_file(
filename,
list(self.preamble) + [self.command],
environ=self.environ,
accept_additional_args=True)
self.exported_command = shell.join(command)
writer.variable(name, self.exported_command)
def get_temporary_directory(self):
"""
Returns a writable temporary directory that is primarily used by loaders
to store temporary files. The temporary directory will be deleted when
the Session context ends unless the ``craftr.keep_temporary_directory``
option is set.
:raise RuntimeError: If the session is not currently in context.
"""
if Session.current is not self:
raise RuntimeError('session not in context')
if not self._tempdir:
self._tempdir = path.join(self.builddir, '.temp')
logger.debug('created temporary directory:', self._tempdir)
return self._tempdir
def relocate_files(files, outdir, suffix, replace_suffix=True, parent=None):
"""
Converts a list of filenames, relocating them to *outdir* and replacing
their existing suffix. If *suffix* is a callable, it will be passed the
new filename and expected to return the same filename, eventually with
a different suffix.
"""
if parent is None:
parent = session.module.project_dir
result = []
for filename in files:
filename = path.join(outdir, path.rel(filename, parent))
filename = path.addsuffix(filename, suffix, replace=replace_suffix)
result.append(filename)
return result
def get_temporary_directory(self):
"""
Returns a writable temporary directory that is primarily used by loaders
to store temporary files. The temporary directory will be deleted when
the Session context ends unless the ``craftr.keep_temporary_directory``
option is set.
:raise RuntimeError: If the session is not currently in context.
"""
if Session.current is not self:
raise RuntimeError('session not in context')
if not self._tempdir:
self._tempdir = path.join(self.builddir, '.temp')
logger.debug('created temporary directory:', self._tempdir)
return self._tempdir
def relocate_files(files, outdir, suffix, replace_suffix=True, parent=None):
"""
Converts a list of filenames, relocating them to *outdir* and replacing
their existing suffix. If *suffix* is a callable, it will be passed the
new filename and expected to return the same filename, eventually with
a different suffix.
"""
if parent is None:
parent = session.module.namespace.project_dir
result = []
for filename in files:
filename = path.join(outdir, path.rel(filename, parent))
filename = path.addsuffix(filename, suffix, replace=replace_suffix)
result.append(filename)
return result
def load_file(filename):
"""
Loads a Python file into a new module-like object and returns it. The
*filename* is assumed relative to the currently executed module's
directory (NOT the project directory which can be different).
"""
if not path.isabs(filename):
filename = path.join(session.module.directory, filename)
with open(filename, 'r') as fp:
code = compile(fp.read(), filename, 'exec')
scope = Namespace()
vars(scope).update(globals())
exec(code, vars(scope))
return scope
def execute(self, parser, args):
if hasattr(args, 'include_path'):
session.path.extend(map(path.norm, args.include_path))
# Help-command preprocessing. Check if we're to show the help on a builtin
# object, otherwise extract the module name if applicable.
if self.mode == 'help':
if not args.name:
help('craftr')
return 0
if args.name in vars(craftr.defaults):
help(getattr(craftr.defaults, args.name))
return 0
# Check if we have an absolute symbol reference.
if ':' in args.name:
if args.module:
parser.error(
'-m/--module option conflicting with name argument: "{}"'
.format(args.name))
args.module, args.name = args.name.split(':', 1)
module = self._find_module(parser, args)
session.main_module = module
self.ninja_bin, self.ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(path.norm(args.build_dir, INIT_DIR))
path.makedirs(session.builddir)
os.chdir(session.builddir)
self.cachefile = path.join(session.builddir, '.craftrcache')
# Prepare options, loaders and execute.
if self.mode in ('export', 'run', 'help'):
return self._export_run_or_help(args, module)
elif self.mode == 'dump-options':
return self._dump_options(args, module)
elif self.mode == 'dump-deptree':
return self._dump_deptree(args, module)
elif self.mode in ('build', 'clean'):
return self._build_or_clean(args)
elif self.mode == 'lock':
self._create_lockfile()
else:
raise RuntimeError("mode: {}".format(self.mode))
def run(self):
"""
Loads the code of the main Craftr build script as specified in the modules
manifest and executes it. Note that this must occur in a context where
the :data:`session` is available.
:raise RuntimeError: If there is no current :data:`session` or if the
module was already executed.
"""
if not session:
raise RuntimeError('no current session')
if self.executed:
raise RuntimeError('already run')
self.executed = True
self.init_options()
self.init_loader()
script_fn = path.norm(path.join(self.directory, self.manifest.main))
with open(script_fn) as fp:
code = compile(fp.read(), script_fn, 'exec')
from craftr import defaults
for key, value in vars(defaults).items():
if not key.startswith('_'):
vars(self.namespace)[key] = value
vars(self.namespace).update({
'__file__': script_fn,
'__name__': self.manifest.name,
'__version__': str(self.manifest.version),
'options': self.options,
'loader': self.loader,
'project_dir': self.project_dir,
})
try:
session.modulestack.append(self)
exec(code, vars(self.namespace))
finally:
assert session.modulestack.pop() is self
def _download_progress(self, url, context, data):
spinning = data["size"] is None
if data["downloaded"] == 0:
# If what we're trying to download already exists, we don't have
# to redownload it.
suffix, directory = self._get_archive_unpack_info(context, data["filename"])
urlfile = path.join(directory, ".craftr_downloadurl")
if path.isfile(urlfile):
with open(urlfile) as fp:
if fp.read().strip() == url:
raise self.DownloadAlreadyExists(directory)
logger.progress_begin("Downloading {}".format(url), spinning)
if spinning:
# TODO: Bytes to human readable
logger.progress_update(None, data["downloaded"])
else:
progress = data["downloaded"] / data["size"]
logger.progress_update(progress, "{}%".format(int(progress * 100)))
if data["completed"]:
logger.progress_end()
def write_response_file(arguments, builder=None, name=None, force_file=False):
"""
Creates a response-file with the specified *name* in the in the
``buildfiles/`` directory and writes the *arguments* list quoted into
the file. If *builder* is specified, it must be a :class:`TargetBuilder`
and the response file will be added to the implicit dependencies.
If *force_file* is set to True, a file will always be written. Otherwise,
the function will into possible limitations of the platform and decide
whether to write a response file or to return the *arguments* as is.
Returns a tuple of ``(filename, arguments)``. If a response file is written,
the returned *arguments* will be a list with a single string that is the
filename prepended with ``@``. The *filename* part can be None if no
response file needed to be exported.
"""
if not name:
if not builder:
raise ValueError('builder must be specified if name is bot')
name = builder.name + '.response.txt'
if platform.name != 'win':
return None, arguments
# We'll just assume that there won't be more than 2048 characters for
# other flags. The windows max buffer size is 8192.
content = shell.join(arguments)
if len(content) < 6144:
return None, arguments
filename = buildlocal(path.join('buildfiles', name))
if builder:
builder.implicit_deps.append(filename)
if session.builddir:
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
fp.write(content)
return filename, ['@' + filename]
def write_response_file(arguments, builder=None, name=None, force_file=False, suffix=''):
"""
Creates a response-file with the specified *name* in the in the
``buildfiles/`` directory and writes the *arguments* list quoted into
the file. If *builder* is specified, it must be a :class:`TargetBuilder`
and the response file will be added to the implicit dependencies.
If *force_file* is set to True, a file will always be written. Otherwise,
the function will into possible limitations of the platform and decide
whether to write a response file or to return the *arguments* as is.
Returns a tuple of ``(filename, arguments)``. If a response file is written,
the returned *arguments* will be a list with a single string that is the
filename prepended with ``@``. The *filename* part can be None if no
response file needed to be exported.
"""
if not name:
if not builder:
raise ValueError('builder must be specified if name is bot')
name = builder.name + suffix + '.response.txt'
if platform.name != 'win':
return None, arguments
# We'll just assume that there won't be more than 2048 characters for
# other flags. The windows max buffer size is 8192.
content = shell.join(arguments)
if len(content) < 6144:
return None, arguments
filename = buildlocal(path.join('buildfiles', name))
if builder:
builder.implicit_deps.append(filename)
if session.builddir:
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
fp.write(content)
return filename, ['@' + filename]
def _find_module(self, parser, args):
"""
Find the main Craftr module that is to be executed. Returns None in
modes that do not require a main module.
"""
if self.mode not in ('export', 'run', 'help', 'dump-options',
'dump-deptree'):
return None
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in MANIFEST_FILENAMES + [
path.join('craftr', x) for x in MANIFEST_FILENAMES
]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
logger.error('"{}" does not exist'.format(
MANIFEST_FILENAMES[0]))
sys.exit(1)
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
logger.error(
'{} (note: you have to escape > and < characters)'.format(
exc))
sys.exit(1)
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
logger.error('module not found: ' + str(exc))
sys.exit(1)
return module
def download_file(url, filename=None, file=None, directory=None,
on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None):
"""
Download a file from a URL to one of the following destinations:
:param filename: A filename to write the downloaded file to.
:param file: A file-like object.
:param directory: A directory. The filename will be automatically
determined from the ``Content-Disposition`` header received from
the server or the last path elemnet in the URL.
Additional parameters for the *directory* parameter:
:param on_exists: The operation to perform when the file already exists.
Available modes are ``rename``, ``overwrite`` and ``skip``.
Additional parameters:
:param progress: A callable that accepts a single parameter that is a
dictionary with information about the progress of the download. The
dictionary provides the keys ``size``, ``downloaded`` and ``response``.
If the callable returns :const:`False` (specifically the value False), the
download will be aborted and a :class:`UserInterrupt` will be raised.
:param urlopen_kwargs: A dictionary with additional keyword arguments
for :func:`urllib.request.urlopen`.
Raise and return:
:raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
:raise URLError: Can be raised by :func:`urllib.request.urlopen`.
:raise UserInterrupt: If the *progress* returned :const:`False`.
:return: If the download mode is *directory*, the name of the downloaded
file will be returned and False if the file was newly downloaded, True
if the download was skipped because the file already existed.
Otherwise, the number of bytes downloaded will be returned.
"""
argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']})
if sum(map(bool, [filename, file, directory])) != 1:
raise ValueError('exactly one of filename, file or directory must be specifed')
response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))
if directory:
try:
filename = parse_content_disposition(
response.headers.get('Content-Disposition', ''))
except ValueError:
filename = url.split('/')[-1]
filename = path.join(directory, filename)
path.makedirs(directory)
if path.exists(filename):
if on_exists == 'skip':
return filename, True
elif on_exists == 'rename':
index = 0
while True:
new_filename = filename + '_{:0>4}'.format(index)
if not path.exists(new_filename):
filename = new_filename
break
index += 1
elif on_exists != 'overwrite':
raise RuntimeError
try:
size = int(response.headers.get('Content-Length', ''))
except ValueError:
size = None
progress_info = {'response': response, 'size': size, 'downloaded': 0,
'completed': False, 'filename': filename, 'url': url}
if progress and progress(progress_info) is False:
raise UserInterrupt
def copy_to_file(fp):
while True:
data = response.read(chunksize)
if not data:
break
progress_info['downloaded'] += len(data)
fp.write(data)
if progress and progress(progress_info) is False:
raise UserInterrupt
progress_info['completed'] = True
if progress and progress(progress_info) is False:
raise UserInterrupt
if filename:
path.makedirs(path.dirname(filename))
try:
with open(filename, 'wb') as fp:
copy_to_file(fp)
except BaseException:
# Delete the file if it could not be downloaded successfully.
path.remove(filename, silent=True)
raise
elif file:
copy_to_file(file)
return filename, False
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.mode == "export":
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error("{} (note: you have to escape > and < characters)".format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error("module not found: " + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, ".craftrcache")
if not read_cache(cachefile) and self.mode != "export":
logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode))
logger.error("Make sure to generate a build tree with 'craftr export'")
return 1
# Prepare options, loaders and execute.
if self.mode == "export":
session.expand_relative_options(module.manifest.name)
session.cache["build"] = {}
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error("error:", exc)
return 1
finally:
if sys.exc_info():
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(cachefile)
# Write the cache back.
session.cache["build"]["targets"] = list(session.graph.targets.keys())
session.cache["build"]["main"] = module.ident
session.cache["build"]["options"] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", "w") as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache["build"]["options"])
main = session.cache["build"]["main"]
available_targets = frozenset(session.cache["build"]["targets"])
logger.debug("build main module:", main)
session.expand_relative_options(get_volatile_module_version(main)[0])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if "." not in target:
target = main + "." + target
elif target.startswith("."):
target = main + target
module_name, target = target.rpartition(".")[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or "*")
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error("no such target: {}".format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ["-v"]
if self.mode == "clean":
cmd += ["-t", "clean"]
if not args.recursive:
cmd += ["-r"]
cmd += targets
return shell.run(cmd).returncode
def main():
# Create argument parsers and dynamically include all BaseCommand
# subclasses into it.
parser = argparse.ArgumentParser(prog="craftr", description="The Craftr build system")
parser.add_argument("-v", "--verbose", action="store_true")
parser.add_argument("-q", "--quiet", action="store_true")
parser.add_argument("-c", "--config", action="append", default=[])
parser.add_argument("-C", "--no-config", action="store_true")
parser.add_argument("-d", "--option", dest="options", action="append", default=[])
subparsers = parser.add_subparsers(dest="command")
commands = {
"clean": BuildCommand("clean"),
"build": BuildCommand("build"),
"export": BuildCommand("export"),
"startpackage": StartpackageCommand(),
"version": VersionCommand(),
}
for key, cmd in commands.items():
cmd.build_parser(subparsers.add_parser(key))
# Parse the arguments.
args = parser.parse_args()
if not args.command:
parser.print_usage()
return 0
if args.verbose:
logger.set_level(logger.DEBUG)
elif args.quiet:
logger.set_level(logger.WARNING)
session = Session()
# Parse the user configuration file.
try:
config_filename = path.expanduser("~/" + CONFIG_FILENAME)
session.options = read_config_file(config_filename)
except FileNotFoundError as exc:
session.options = {}
except InvalidConfigError as exc:
parser.error(exc)
return 1
# Parse the local configuration file or the ones specified on command-line.
if not args.no_config:
try:
for filename in args.config:
session.options.update(read_config_file(filename))
if not args.config:
choices = [CONFIG_FILENAME, path.join("craftr", CONFIG_FILENAME)]
for fn in choices:
try:
session.options.update(read_config_file(fn))
except FileNotFoundError as exc:
pass
except InvalidConfigError as exc:
parser.error(exc)
return 1
# Execute the command in the session context.
with session:
parse_cmdline_options(args.options)
return commands[args.command].execute(parser, args)
def download_file(url,
filename=None,
file=None,
directory=None,
on_exists='rename',
progress=None,
chunksize=4096,
urlopen_kwargs=None):
"""
Download a file from a URL to one of the following destinations:
:param filename: A filename to write the downloaded file to.
:param file: A file-like object.
:param directory: A directory. The filename will be automatically
determined from the ``Content-Disposition`` header received from
the server or the last path elemnet in the URL.
Additional parameters for the *directory* parameter:
:param on_exists: The operation to perform when the file already exists.
Available modes are ``rename``, ``overwrite`` and ``skip``.
Additional parameters:
:param progress: A callable that accepts a single parameter that is a
dictionary with information about the progress of the download. The
dictionary provides the keys ``size``, ``downloaded`` and ``response``.
If the callable returns :const:`False` (specifically the value False), the
download will be aborted and a :class:`UserInterrupt` will be raised.
:param urlopen_kwargs: A dictionary with additional keyword arguments
for :func:`urllib.request.urlopen`.
Raise and return:
:raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
:raise URLError: Can be raised by :func:`urllib.request.urlopen`.
:raise UserInterrupt: If the *progress* returned :const:`False`.
:return: If the download mode is *directory*, the name of the downloaded
file will be returned and False if the file was newly downloaded, True
if the download was skipped because the file already existed.
Otherwise, the number of bytes downloaded will be returned.
"""
argspec.validate('on_exists', on_exists,
{'enum': ['rename', 'overwrite', 'skip']})
if sum(map(bool, [filename, file, directory])) != 1:
raise ValueError(
'exactly one of filename, file or directory must be specifed')
response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))
if directory:
try:
filename = parse_content_disposition(
response.headers.get('Content-Disposition', ''))
except ValueError:
filename = url.split('/')[-1]
filename = path.join(directory, filename)
path.makedirs(directory)
if path.exists(filename):
if on_exists == 'skip':
return filename, True
elif on_exists == 'rename':
index = 0
while True:
new_filename = filename + '_{:0>4}'.format(index)
if not path.exists(new_filename):
filename = new_filename
break
index += 1
elif on_exists != 'overwrite':
raise RuntimeError
try:
size = int(response.headers.get('Content-Length', ''))
except ValueError:
size = None
progress_info = {
'response': response,
'size': size,
'downloaded': 0,
'completed': False,
'filename': filename,
'url': url
}
if progress and progress(progress_info) is False:
raise UserInterrupt
def copy_to_file(fp):
while True:
data = response.read(chunksize)
if not data:
break
progress_info['downloaded'] += len(data)
fp.write(data)
if progress and progress(progress_info) is False:
raise UserInterrupt
progress_info['completed'] = True
if progress and progress(progress_info) is False:
raise UserInterrupt
if filename:
path.makedirs(path.dirname(filename))
try:
with open(filename, 'wb') as fp:
copy_to_file(fp)
except BaseException:
# Delete the file if it could not be downloaded successfully.
path.remove(filename, silent=True)
raise
elif file:
copy_to_file(file)
return filename, False
def main():
# Create argument parsers and dynamically include all BaseCommand
# subclasses into it.
parser = argparse.ArgumentParser(prog='craftr', description='The Craftr build system')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('-q', '--quiet', action='store_true')
parser.add_argument('-c', '--config', action='append', default=[])
parser.add_argument('-C', '--no-config', action='store_true')
parser.add_argument('-d', '--option', dest='options', action='append', default=[])
subparsers = parser.add_subparsers(dest='command')
commands = {
'export': ExportOrBuildCommand(is_export=True),
'build': ExportOrBuildCommand(is_export=False),
'startpackage': StartpackageCommand()
}
for key, cmd in commands.items():
cmd.build_parser(subparsers.add_parser(key))
# Parse the arguments.
args = parser.parse_args()
if not args.command:
parser.print_usage()
return 0
if args.verbose:
logger.set_level(logger.DEBUG)
elif args.quiet:
logger.set_level(logger.WARNING)
session = Session()
# Parse the user configuration file.
try:
config_filename = path.expanduser('~/' + CONFIG_FILENAME)
session.options = read_config_file(config_filename)
except FileNotFoundError as exc:
session.options = {}
except InvalidConfigError as exc:
parser.error(exc)
return 1
# Parse the local configuration file or the ones specified on command-line.
if not args.no_config:
try:
for filename in args.config:
session.options.update(read_config_file(filename))
if not args.config:
choices = [CONFIG_FILENAME, path.join('craftr', CONFIG_FILENAME)]
for fn in choices:
try:
session.options.update(read_config_file(fn))
except FileNotFoundError as exc:
pass
except InvalidConfigError as exc:
parser.error(exc)
return 1
# Execute the command in the session context.
with session:
parse_cmdline_options(args.options)
return commands[args.command].execute(parser, args)
def project_dir(self):
return path.norm(path.join(self.directory, self.manifest.project_dir))
def scriptfile(self):
return path.norm(path.join(self.directory, self.manifest.main))
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.is_export:
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join('craftr', MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error('{} (note: you have to escape > and < characters)'.format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error('module not found: ' + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, '.craftrcache')
if not read_cache(cachefile) and not self.is_export:
logger.error('Unable to load "{}", can not build'.format(cachefile))
return 1
# Prepare options, loaders and execute.
if self.is_export:
session.cache['build'] = {}
try:
write_cache(cachefile)
module.run()
except (Module.InvalidOption, Module.LoaderInitializationError) as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error(exc)
return 1
# Write the cache back.
session.cache['build']['targets'] = list(session.graph.targets.keys())
session.cache['build']['main'] = module.ident
session.cache['build']['options'] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", 'w') as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache['build']['options'])
main = session.cache['build']['main']
available_targets = frozenset(session.cache['build']['targets'])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if '.' not in target:
target = main + '.' + target
elif target.startswith('.'):
target = main + target
module_name, target = target.rpartition('.')[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or '*')
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error('no such target: {}'.format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ['-v']
cmd += targets
shell.run(cmd)
def __call__(self, value):
from craftr.core.session import session
if not path.isabs(value):
value = path.join(session.maindir, value)
return path.norm(value)
def __call__(self, value):
from craftr.core.session import session
if not path.isabs(value):
value = path.join(session.maindir, value)
return path.norm(value)
def project_dir(self): return path.norm(path.join(self.directory, self.manifest.project_dir))
def _get_archive_unpack_info(self, context, archive):
suffix = nr.misc.archive.get_opener(archive)[0]
filename = path.basename(archive)[: -len(suffix)]
directory = path.join(context.installdir, filename)
return suffix, directory
def _export_run_or_help(self, args, module):
"""
Called when the mode is 'export' or 'run'. Will execute the specified
*module* and eventually export a Ninja manifest and Cache.
"""
read_cache(False)
session.expand_relative_options()
session.cache['build'] = {}
# Load the dependency lock information if it exists.
deplock_fn = path.join(path.dirname(module.manifest.filename),
'.dependency-lock')
if os.path.isfile(deplock_fn):
with open(deplock_fn) as fp:
session.preferred_versions = cson.load(fp)
logger.debug('note: dependency lock file "{}" loaded'.format(
deplock_fn))
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error('error:', exc)
return 1
finally:
if sys.exc_info() and self.mode == 'export':
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(self.cachefile)
# Fill the cache.
session.cache['build']['targets'] = list(session.graph.targets.keys())
session.cache['build']['modules'] = serialise_loaded_module_info()
session.cache['build']['main'] = module.ident
session.cache['build']['options'] = args.options
session.cache['build']['dependency_lock_filename'] = deplock_fn
if self.mode == 'export':
# Add the Craftr_run_command variable which is necessary for tasks
# to properly executed.
run_command = ['craftr', '-q', '-P', path.rel(session.maindir)]
if args.no_config: run_command += ['-C']
run_command += ['-c' + x for x in args.config]
run_command += ['run']
if args.module: run_command += ['-m', args.module]
run_command += ['-i' + x for x in args.include_path]
run_command += ['-b', path.rel(session.builddir)]
session.graph.vars['Craftr_run_command'] = shell.join(run_command)
write_cache(self.cachefile)
# Write the Ninja manifest.
with open("build.ninja", 'w') as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(self.ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
logger.info('exported "build.ninja"')
return 0
elif self.mode == 'run':
if args.task:
if args.task not in session.graph.tasks:
logger.error('no such task exists: "{}"'.format(args.task))
return 1
task = session.graph.tasks[args.task]
return task.invoke(args.task_args)
return 0
elif self.mode == 'help':
if args.name not in vars(module.namespace):
logger.error('symbol not found: "{}:{}"'.format(
module.manifest.name, args.name))
return 1
help(getattr(module.namespace, args.name))
return 0
assert False, "unhandled mode: {}".format(self.mode)
def load(self, context, cache):
if cache is not None and path.isdir(cache.get("directory", "")):
# Check if the requested version changes.
url_template = context.expand_variables(cache.get("url_template", ""))
if url_template == cache.get("url"):
self.directory = cache["directory"]
logger.info("Reusing cached directory: {}".format(path.rel(self.directory, nopar=True)))
return cache
else:
logger.info("Cached URL is outdated:", cache.get("url"))
directory = None
archive = None
delete_after_extract = True
for url_template in self.urls:
url = context.expand_variables(url_template)
if not url:
continue
if url.startswith("file://"):
name = url[7:]
if path.isdir(name):
logger.info("Using directory", url)
directory = name
break
elif path.isfile(name):
logger.info("Using archive", url)
archive = name
delete_after_extract = False
break
error = None
else:
error = None
try:
progress = lambda d: self._download_progress(url, context, d)
archive, reused = httputils.download_file(
url, directory=context.get_temporary_directory(), on_exists="skip", progress=progress
)
except (httputils.URLError, httputils.HTTPError) as exc:
error = exc
except self.DownloadAlreadyExists as exc:
directory = exc.directory
logger.info("Reusing existing directory", directory)
else:
if reused:
logger.info("Reusing cached download", path.basename(archive))
break
if error:
logger.info("Error reading", url, ":", error)
if directory or archive:
logger.debug("URL applies: {}".format(url))
if not directory and archive:
suffix, directory = self._get_archive_unpack_info(context, archive)
logger.info(
'Unpacking "{}" to "{}" ...'.format(path.rel(archive, nopar=True), path.rel(directory, nopar=True))
)
nr.misc.archive.extract(
archive,
directory,
suffix=suffix,
unpack_single_dir=True,
check_extract_file=self._check_extract_file,
progress_callback=self._extract_progress,
)
elif not directory:
raise LoaderError(self, "no URL matched")
self.directory = directory
with open(path.join(self.directory, ".craftr_downloadurl"), "w") as fp:
fp.write(url)
return {"directory": directory, "url_template": url_template, "url": url}
def main():
# Create argument parsers and dynamically include all BaseCommand
# subclasses into it.
parser = argparse.ArgumentParser(prog='craftr',
description='The Craftr build system')
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('-q', '--quiet', action='store_true')
parser.add_argument('-c', '--config', action='append', default=[])
parser.add_argument('-C', '--no-config', action='store_true')
parser.add_argument('-P', '--project-dir')
parser.add_argument('-d',
'--option',
dest='options',
action='append',
default=[])
subparsers = parser.add_subparsers(dest='command')
commands = {
'lock': BuildCommand('lock'),
'clean': BuildCommand('clean'),
'build': BuildCommand('build'),
'export': BuildCommand('export'),
'run': BuildCommand('run'),
'help': BuildCommand('help'),
'options': BuildCommand('dump-options'),
'deptree': BuildCommand('dump-deptree'),
'startpackage': StartpackageCommand(),
'version': VersionCommand()
}
for key, cmd in commands.items():
cmd.build_parser(subparsers.add_parser(key))
# Parse the arguments.
args = parser.parse_args()
if not args.command:
parser.print_usage()
return 0
if args.project_dir:
os.chdir(args.project_dir)
if args.verbose:
logger.set_level(logger.DEBUG)
elif args.quiet:
logger.set_level(logger.WARNING)
session = Session()
# Parse the user configuration file.
try:
config_filename = path.expanduser('~/' + CONFIG_FILENAME)
session.options = read_config_file(config_filename)
except FileNotFoundError as exc:
session.options = {}
except InvalidConfigError as exc:
parser.error(exc)
return 1
# Parse the local configuration file or the ones specified on command-line.
if not args.no_config:
try:
for filename in args.config:
session.options.update(read_config_file(filename))
if not args.config:
choices = [
CONFIG_FILENAME,
path.join('craftr', CONFIG_FILENAME)
]
for fn in choices:
try:
session.options.update(read_config_file(fn))
except FileNotFoundError as exc:
pass
except InvalidConfigError as exc:
parser.error(exc)
return 1
# Execute the command in the session context.
with session:
parse_cmdline_options(args.options)
return commands[args.command].execute(parser, args)
class Session(object):
"""
This class manages the :class:`build.Graph` and loading of Craftr modules.
.. attribute:: graph
A :class:`build.Graph` instance.
.. attribute:: path
A list of paths that will be searched for Craftr modules.
.. attribute:: module
The Craftr module that is currently being executed. This is an instance
of the :class:`Module` class and the same as the tip of the
:attr:`modulestack`.
.. attribute:: modulestack
A list of modules where the last element (tip) is the module that is
currently being executed.
.. attribute:: modules
A nested dictionary that maps from name to a dictionary of version
numbers mapping to :class:`Module` objects. These are the modules that
have already been loaded into the session or that have been found and
cached but not yet been executed.
.. attribute:: preferred_versions
A nested dictionary with the same structure as :attr:`modules`. This
dictionary might have been loaded from a dependency lock file and specifies
the preferred version to load for a specific module, assuming that the
criteria specified in the loading module's manifest is less strict. Note
that Craftr will error if a preferred version can not be found.
.. attribute:: maindir
The main directory from which Craftr was run. Craftr will switch to the
build directory at a later point, which is why we keep this member for
reference.
.. attribute:: builddir
The absolute path to the build directory.
.. attribute:: main_module
The main :class:`Module`.
.. attribute:: options
A dictionary of options that are passed down to Craftr modules.
.. attributes:: cache
A JSON object that will be loaded from the current workspace's cache
file and written back when Craftr exits without errors. The cache can
contain anything and can be modified by everything, however it should
be assured that no name conflicts and accidental modifications/deletes
occur.
Reserved keywords in the cache are ``"build"`` and ``"loaders"``.
"""
#: The current session object. Create it with :meth:`start` and destroy
#: it with :meth:`end`.
current = None
#: Diretory that contains the Craftr standard library.
stl_dir = path.norm(path.join(__file__, '../../stl'))
stl_auxiliary_dir = path.norm(path.join(__file__, '../../stl_auxiliary'))
def __init__(self, maindir=None):
self.maindir = path.norm(maindir or path.getcwd())
self.builddir = path.join(self.maindir, 'build')
self.graph = build.Graph()
self.path = [
self.stl_dir, self.stl_auxiliary_dir, self.maindir,
path.join(self.maindir, 'craftr/modules')
]
self.modulestack = []
self.modules = {}
self.preferred_versions = {}
self.main_module = None
self.options = {}
self.cache = {}
self.tasks = {}
self._tempdir = None
self._manifest_cache = {} # maps manifest_filename: manifest
self._refresh_cache = True
def __enter__(self):
if Session.current:
raise RuntimeError('a session was already created')
Session.current = self
return Session.current
def __exit__(self, exc_value, exc_type, exc_tb):
if Session.current is not self:
raise RuntimeError('session not in context')
if self._tempdir and not self.options.get(
'craftr.keep_temporary_directory'):
logger.debug('removing temporary directory:', self._tempdir)
try:
path.remove(self._tempdir, recursive=True)
except OSError as exc:
logger.debug('error:', exc, indent=1)
finally:
self._tempdir = None
Session.current = None
@property
def module(self):
if self.modulestack:
return self.modulestack[-1]
return None
def read_cache(self, fp):
cache = json.load(fp)
if not isinstance(cache, dict):
raise ValueError(
'Craftr Session cache must be a JSON object, got {}'.format(
type(cache).__name__))
self.cache = cache
def write_cache(self, fp):
json.dump(self.cache, fp, indent='\t')
def expand_relative_options(self, module_name=None):
"""
After the main module has been detected, relative option names (starting
with ``.``) should be converted to absolute option names. This is what
the method does.
"""
if not module_name and not self.main_module:
raise RuntimeError('main_module not set')
if not module_name:
module_name = self.main_module.manifest.name
for key in tuple(self.options.keys()):
if key.startswith('.'):
self.options[module_name + key] = self.options.pop(key)
def get_temporary_directory(self):
"""
Returns a writable temporary directory that is primarily used by loaders
to store temporary files. The temporary directory will be deleted when
the Session context ends unless the ``craftr.keep_temporary_directory``
option is set.
:raise RuntimeError: If the session is not currently in context.
"""
if Session.current is not self:
raise RuntimeError('session not in context')
if not self._tempdir:
self._tempdir = path.join(self.builddir, '.temp')
logger.debug('created temporary directory:', self._tempdir)
return self._tempdir
def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
other = versions[manifest.version].manifest.filename
logger.debug('multiple occurences of "{}-{}" found\n'
' - {}\n - {}'.format(manifest.name,
manifest.version, filename,
other))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def update_manifest_cache(self, force=False):
if not self._refresh_cache and not force:
return
self._refresh_cache = False
for directory in self.path:
choices = []
choices.extend(
[path.join(directory, x) for x in MANIFEST_FILENAMES])
for item in path.easy_listdir(directory):
choices.extend([
path.join(directory, item, x) for x in MANIFEST_FILENAMES
])
choices.extend([
path.join(directory, item, 'craftr', x)
for x in MANIFEST_FILENAMES
])
for filename in map(path.norm, choices):
if filename in self._manifest_cache:
continue # don't parse a manifest that we already parsed
if not path.isfile(filename):
continue
try:
self.parse_manifest(filename)
except Manifest.Invalid as exc:
logger.warn('invalid manifest found:', filename)
logger.warn(exc, indent=1)
def find_module(self, name, version, resolve_preferred_version=True):
"""
Finds a module in the :attr:`path` matching the specified *name* and
*version*.
:param name: The name of the module.
:param version: A :class:`VersionCriteria`, :class:`Version` or string
in a VersionCritiera format.
:param resolve_preferred_version: If this parameter is True (default)
and a preferred version is specified in :attr:`preferred_versions`,
that preferred version is loaded or :class:`ModuleNotFound` is raised.
:raise ModuleNotFound: If the module can not be found.
:return: :class:`Module`
"""
argspec.validate('name', name, {'type': str})
argspec.validate('version', version,
{'type': [str, Version, VersionCriteria]})
if name in renames.renames:
logger.warn('"{}" is deprecated, use "{}" instead'.format(
name, renames.renames[name]))
name = renames.renames[name]
if isinstance(version, str):
try:
version = Version(version)
except ValueError as exc:
version = VersionCriteria(version)
if session.module and resolve_preferred_version:
data = self.preferred_versions.get(session.module.manifest.name)
if data is not None:
versions = data.get(str(session.module.manifest.version))
if versions is not None:
preferred_version = versions.get(name)
if preferred_version is not None:
version = Version(preferred_version)
logger.debug(
'note: loading preferred version {} of module "{}" '
'requested by module "{}"'.format(
version, name, session.module.ident))
self.update_manifest_cache()
if name in self.modules:
if isinstance(version, Version):
if version in self.modules[name]:
return self.modules[name][version]
raise ModuleNotFound(name, version)
for module in sorted(self.modules[name].values(),
key=lambda x: x.manifest.version,
reverse=True):
if version(module.manifest.version):
return module
raise ModuleNotFound(name, version)
