def update_manifest_cache(self, force=False):
if not self._refresh_cache and not force:
return
self._refresh_cache = False
for directory in self.path:
choices = []
choices.extend(
[path.join(directory, x) for x in MANIFEST_FILENAMES])
for item in path.easy_listdir(directory):
choices.extend([
path.join(directory, item, x) for x in MANIFEST_FILENAMES
])
choices.extend([
path.join(directory, item, 'craftr', x)
for x in MANIFEST_FILENAMES
])
for filename in map(path.norm, choices):
if filename in self._manifest_cache:
continue # don't parse a manifest that we already parsed
if not path.isfile(filename):
continue
try:
self.parse_manifest(filename)
except Manifest.Invalid as exc:
logger.warn('invalid manifest found:', filename)
logger.warn(exc, indent=1)
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, 'craftr')
path.makedirs(directory)
mfile = path.join(directory, 'manifest.' + args.format)
sfile = path.join(directory, 'Craftrfile')
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, 'w') as fp:
if args.format == 'cson':
lines = textwrap.dedent('''
name: "%s"
version: "%s"
project_dir: ".."
author: ""
url: ""
dependencies: {}
options: {}
''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[2]
elif args.format == 'json':
lines = textwrap.dedent('''
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[3]
fp.write('\n'.join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, 'w') as fp:
print('# {}'.format(args.name), file=fp)
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, "craftr")
path.makedirs(directory)
mfile = path.join(directory, MANIFEST_FILENAME)
sfile = path.join(directory, "Craftrfile")
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, "w") as fp:
lines = (
textwrap.dedent(
"""
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}\n"""
% (args.name, args.version)
)
.lstrip()
.split("\n")
)
if not args.nested:
del lines[3]
fp.write("\n".join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, "w") as fp:
print("# {}".format(args.name), file=fp)
def _download_progress(self, url, context, data):
spinning = data["size"] is None
if data["downloaded"] == 0:
# If what we're trying to download already exists, we don't have
# to redownload it.
suffix, directory = self._get_archive_unpack_info(context, data["filename"])
urlfile = path.join(directory, ".craftr_downloadurl")
if path.isfile(urlfile):
with open(urlfile) as fp:
if fp.read().strip() == url:
raise self.DownloadAlreadyExists(directory)
logger.progress_begin("Downloading {}".format(url), spinning)
if spinning:
# TODO: Bytes to human readable
logger.progress_update(None, data["downloaded"])
else:
progress = data["downloaded"] / data["size"]
logger.progress_update(progress, "{}%".format(int(progress * 100)))
if data["completed"]:
logger.progress_end()
def _find_module(self, parser, args):
"""
Find the main Craftr module that is to be executed. Returns None in
modes that do not require a main module.
"""
if self.mode not in ('export', 'run', 'help', 'dump-options',
'dump-deptree'):
return None
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in MANIFEST_FILENAMES + [
path.join('craftr', x) for x in MANIFEST_FILENAMES
]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
logger.error('"{}" does not exist'.format(
MANIFEST_FILENAMES[0]))
sys.exit(1)
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
logger.error(
'{} (note: you have to escape > and < characters)'.format(
exc))
sys.exit(1)
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
logger.error('module not found: ' + str(exc))
sys.exit(1)
return module
def update_manifest_cache(self, force=False):
if not self._refresh_cache and not force:
return
self._refresh_cache = False
for directory in self.path:
choices = []
choices.append(path.join(directory, MANIFEST_FILENAME))
for item in path.easy_listdir(directory):
choices.append(path.join(directory, item, MANIFEST_FILENAME))
choices.append(path.join(directory, item, 'craftr', MANIFEST_FILENAME))
for filename in map(path.norm, choices):
if filename in self._manifest_cache:
continue # don't parse a manifest that we already parsed
if not path.isfile(filename):
continue
try:
self.parse_manifest(filename)
except Manifest.Invalid as exc:
logger.warn('invalid manifest found:', filename)
logger.warn(exc, indent=1)
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug("reading configuration file:", filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename))
continue
elif section == "__global__":
prefix = ""
else:
prefix = section + "."
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.is_export:
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join('craftr', MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error('{} (note: you have to escape > and < characters)'.format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error('module not found: ' + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, '.craftrcache')
if not read_cache(cachefile) and not self.is_export:
logger.error('Unable to load "{}", can not build'.format(cachefile))
return 1
# Prepare options, loaders and execute.
if self.is_export:
session.cache['build'] = {}
try:
write_cache(cachefile)
module.run()
except (Module.InvalidOption, Module.LoaderInitializationError) as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error(exc)
return 1
# Write the cache back.
session.cache['build']['targets'] = list(session.graph.targets.keys())
session.cache['build']['main'] = module.ident
session.cache['build']['options'] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", 'w') as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache['build']['options'])
main = session.cache['build']['main']
available_targets = frozenset(session.cache['build']['targets'])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if '.' not in target:
target = main + '.' + target
elif target.startswith('.'):
target = main + target
module_name, target = target.rpartition('.')[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or '*')
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error('no such target: {}'.format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ['-v']
cmd += targets
shell.run(cmd)
def load(self, context, cache):
if cache is not None and path.isdir(cache.get("directory", "")):
# Check if the requested version changes.
url_template = context.expand_variables(cache.get("url_template", ""))
if url_template == cache.get("url"):
self.directory = cache["directory"]
logger.info("Reusing cached directory: {}".format(path.rel(self.directory, nopar=True)))
return cache
else:
logger.info("Cached URL is outdated:", cache.get("url"))
directory = None
archive = None
delete_after_extract = True
for url_template in self.urls:
url = context.expand_variables(url_template)
if not url:
continue
if url.startswith("file://"):
name = url[7:]
if path.isdir(name):
logger.info("Using directory", url)
directory = name
break
elif path.isfile(name):
logger.info("Using archive", url)
archive = name
delete_after_extract = False
break
error = None
else:
error = None
try:
progress = lambda d: self._download_progress(url, context, d)
archive, reused = httputils.download_file(
url, directory=context.get_temporary_directory(), on_exists="skip", progress=progress
)
except (httputils.URLError, httputils.HTTPError) as exc:
error = exc
except self.DownloadAlreadyExists as exc:
directory = exc.directory
logger.info("Reusing existing directory", directory)
else:
if reused:
logger.info("Reusing cached download", path.basename(archive))
break
if error:
logger.info("Error reading", url, ":", error)
if directory or archive:
logger.debug("URL applies: {}".format(url))
if not directory and archive:
suffix, directory = self._get_archive_unpack_info(context, archive)
logger.info(
'Unpacking "{}" to "{}" ...'.format(path.rel(archive, nopar=True), path.rel(directory, nopar=True))
)
nr.misc.archive.extract(
archive,
directory,
suffix=suffix,
unpack_single_dir=True,
check_extract_file=self._check_extract_file,
progress_callback=self._extract_progress,
)
elif not directory:
raise LoaderError(self, "no URL matched")
self.directory = directory
with open(path.join(self.directory, ".craftr_downloadurl"), "w") as fp:
fp.write(url)
return {"directory": directory, "url_template": url_template, "url": url}
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.mode == "export":
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error("{} (note: you have to escape > and < characters)".format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error("module not found: " + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, ".craftrcache")
if not read_cache(cachefile) and self.mode != "export":
logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode))
logger.error("Make sure to generate a build tree with 'craftr export'")
return 1
# Prepare options, loaders and execute.
if self.mode == "export":
session.expand_relative_options(module.manifest.name)
session.cache["build"] = {}
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error("error:", exc)
return 1
finally:
if sys.exc_info():
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(cachefile)
# Write the cache back.
session.cache["build"]["targets"] = list(session.graph.targets.keys())
session.cache["build"]["main"] = module.ident
session.cache["build"]["options"] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", "w") as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache["build"]["options"])
main = session.cache["build"]["main"]
available_targets = frozenset(session.cache["build"]["targets"])
logger.debug("build main module:", main)
session.expand_relative_options(get_volatile_module_version(main)[0])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if "." not in target:
target = main + "." + target
elif target.startswith("."):
target = main + target
module_name, target = target.rpartition(".")[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or "*")
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error("no such target: {}".format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ["-v"]
if self.mode == "clean":
cmd += ["-t", "clean"]
if not args.recursive:
cmd += ["-r"]
cmd += targets
return shell.run(cmd).returncode
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug('reading configuration file:', filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError(
'file "{}" included by "{}" does not exist'.format(
str(exc), filename))
continue
elif section == '__global__':
prefix = ''
else:
prefix = section + '.'
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
