def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
logger.debug('multiple occurences of "{}-{}" found, '
'one of which is located at "{}"'.format(manifest.name,
manifest.version, filename))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def write_command_file(self, filename, commands, inputs=None, outputs=None,
cwd=None, environ=None, foreach=False, suffix='.cmd', dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = ['cmd', '/Q', '/c', filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(path.abs(filename)))
with open(filename, 'w') as fp:
fp.write('REM This file is automatically generated with Craftr. It is \n')
fp.write('REM not recommended to modify it manually.\n\n')
if cwd is not None:
fp.write('cd ' + shell.quote(cwd) + '\n\n')
for key, value in environ.items():
fp.write('set ' + shell.quote('{}={}'.format(key, value), for_ninja=True) + '\n')
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands)-1:
command.append(shell.safe('%*'))
fp.write(shell.join(command) + '\n')
fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n')
return result, filename
def parse_manifest(self, filename):
"""
Parse a manifest by filename and add register the module to the module
cache. Returns the :class:`Module` object. If the manifest has already
been parsed, it will not be re-parsed.
:raise Manifest.Invalid: If the manifest is invalid.
:return: :const:`None` if the manifest is a duplicate of an already
parsed manifest (determined by name and version), otherwise the
:class:`Module` object for the manifest's module.
"""
filename = path.norm(path.abs(filename))
if filename in self._manifest_cache:
manifest = self._manifest_cache[filename]
return self.find_module(manifest.name, manifest.version)
manifest = Manifest.parse(filename)
self._manifest_cache[filename] = manifest
versions = self.modules.setdefault(manifest.name, {})
if manifest.version in versions:
other = versions[manifest.version].manifest.filename
logger.debug('multiple occurences of "{}-{}" found\n'
' - {}\n - {}'.format(manifest.name,
manifest.version, filename,
other))
module = None
else:
logger.debug('parsed manifest: {}-{} ({})'.format(
manifest.name, manifest.version, filename))
module = Module(path.dirname(filename), manifest)
versions[manifest.version] = module
return module
def write_cache(cachefile):
# Write back the cache.
try:
path.makedirs(path.dirname(cachefile))
with open(cachefile, 'w') as fp:
session.write_cache(fp)
except OSError as exc:
logger.error('error writing cache file:', cachefile)
logger.error(exc, indent=1)
else:
logger.debug('cache written:', cachefile)
def write_cache(cachefile):
# Write back the cache.
try:
path.makedirs(path.dirname(cachefile))
with open(cachefile, 'w') as fp:
session.write_cache(fp)
except OSError as exc:
logger.error('error writing cache file:', cachefile)
logger.error(exc, indent=1)
else:
logger.debug('cache written:', cachefile)
def write_command_file(self,
filename,
commands,
inputs=None,
outputs=None,
cwd=None,
environ=None,
foreach=False,
suffix='.sh',
dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = [filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
# TODO: Make sure this also works for shells other than bash.
fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) +
'\n')
fp.write('set -e\n')
if cwd:
fp.write('cd ' + shell.quote(cwd) + '\n')
fp.write('\n')
for key, value in environ.items():
fp.write('export {}={}\n'.format(key, shell.quote(value)))
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands) - 1:
command.append(shell.safe('$*'))
fp.write(shell.join(command))
fp.write('\n')
os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # rwxrw-r--
return result, filename
def write_command_file(self,
filename,
commands,
inputs=None,
outputs=None,
cwd=None,
environ=None,
foreach=False,
suffix='.cmd',
dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = ['cmd', '/Q', '/c', filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(path.abs(filename)))
with open(filename, 'w') as fp:
fp.write(
'REM This file is automatically generated with Craftr. It is \n'
)
fp.write('REM not recommended to modify it manually.\n\n')
if cwd is not None:
fp.write('cd ' + shell.quote(cwd) + '\n\n')
for key, value in environ.items():
fp.write(
'set ' +
shell.quote('{}={}'.format(key, value), for_ninja=True) +
'\n')
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands) - 1:
command.append(shell.safe('%*'))
fp.write(shell.join(command) + '\n')
fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n')
return result, filename
def write_response_file(arguments, builder=None, name=None, force_file=False, suffix=''):
"""
Creates a response-file with the specified *name* in the in the
``buildfiles/`` directory and writes the *arguments* list quoted into
the file. If *builder* is specified, it must be a :class:`TargetBuilder`
and the response file will be added to the implicit dependencies.
If *force_file* is set to True, a file will always be written. Otherwise,
the function will into possible limitations of the platform and decide
whether to write a response file or to return the *arguments* as is.
Returns a tuple of ``(filename, arguments)``. If a response file is written,
the returned *arguments* will be a list with a single string that is the
filename prepended with ``@``. The *filename* part can be None if no
response file needed to be exported.
"""
if not name:
if not builder:
raise ValueError('builder must be specified if name is bot')
name = builder.name + suffix + '.response.txt'
if platform.name != 'win':
return None, arguments
# We'll just assume that there won't be more than 2048 characters for
# other flags. The windows max buffer size is 8192.
content = shell.join(arguments)
if len(content) < 6144:
return None, arguments
filename = buildlocal(path.join('buildfiles', name))
if builder:
builder.implicit_deps.append(filename)
if session.builddir:
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
fp.write(content)
return filename, ['@' + filename]
def write_response_file(arguments, builder=None, name=None, force_file=False):
"""
Creates a response-file with the specified *name* in the in the
``buildfiles/`` directory and writes the *arguments* list quoted into
the file. If *builder* is specified, it must be a :class:`TargetBuilder`
and the response file will be added to the implicit dependencies.
If *force_file* is set to True, a file will always be written. Otherwise,
the function will into possible limitations of the platform and decide
whether to write a response file or to return the *arguments* as is.
Returns a tuple of ``(filename, arguments)``. If a response file is written,
the returned *arguments* will be a list with a single string that is the
filename prepended with ``@``. The *filename* part can be None if no
response file needed to be exported.
"""
if not name:
if not builder:
raise ValueError('builder must be specified if name is bot')
name = builder.name + '.response.txt'
if platform.name != 'win':
return None, arguments
# We'll just assume that there won't be more than 2048 characters for
# other flags. The windows max buffer size is 8192.
content = shell.join(arguments)
if len(content) < 6144:
return None, arguments
filename = buildlocal(path.join('buildfiles', name))
if builder:
builder.implicit_deps.append(filename)
if session.builddir:
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
fp.write(content)
return filename, ['@' + filename]
def write_command_file(self, filename, commands, inputs=None, outputs=None,
cwd=None, environ=None, foreach=False, suffix='.sh', dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = [filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
# TODO: Make sure this also works for shells other than bash.
fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) + '\n')
fp.write('set -e\n')
if cwd:
fp.write('cd ' + shell.quote(cwd) + '\n')
fp.write('\n')
for key, value in environ.items():
fp.write('export {}={}\n'.format(key, shell.quote(value)))
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands)-1:
command.append(shell.safe('$*'))
fp.write(shell.join(command))
fp.write('\n')
os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # rwxrw-r--
return result, filename
def _export_run_or_help(self, args, module):
"""
Called when the mode is 'export' or 'run'. Will execute the specified
*module* and eventually export a Ninja manifest and Cache.
"""
read_cache(False)
session.expand_relative_options()
session.cache['build'] = {}
# Load the dependency lock information if it exists.
deplock_fn = path.join(path.dirname(module.manifest.filename),
'.dependency-lock')
if os.path.isfile(deplock_fn):
with open(deplock_fn) as fp:
session.preferred_versions = cson.load(fp)
logger.debug('note: dependency lock file "{}" loaded'.format(
deplock_fn))
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error('error:', exc)
return 1
finally:
if sys.exc_info() and self.mode == 'export':
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(self.cachefile)
# Fill the cache.
session.cache['build']['targets'] = list(session.graph.targets.keys())
session.cache['build']['modules'] = serialise_loaded_module_info()
session.cache['build']['main'] = module.ident
session.cache['build']['options'] = args.options
session.cache['build']['dependency_lock_filename'] = deplock_fn
if self.mode == 'export':
# Add the Craftr_run_command variable which is necessary for tasks
# to properly executed.
run_command = ['craftr', '-q', '-P', path.rel(session.maindir)]
if args.no_config: run_command += ['-C']
run_command += ['-c' + x for x in args.config]
run_command += ['run']
if args.module: run_command += ['-m', args.module]
run_command += ['-i' + x for x in args.include_path]
run_command += ['-b', path.rel(session.builddir)]
session.graph.vars['Craftr_run_command'] = shell.join(run_command)
write_cache(self.cachefile)
# Write the Ninja manifest.
with open("build.ninja", 'w') as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(self.ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
logger.info('exported "build.ninja"')
return 0
elif self.mode == 'run':
if args.task:
if args.task not in session.graph.tasks:
logger.error('no such task exists: "{}"'.format(args.task))
return 1
task = session.graph.tasks[args.task]
return task.invoke(args.task_args)
return 0
elif self.mode == 'help':
if args.name not in vars(module.namespace):
logger.error('symbol not found: "{}:{}"'.format(
module.manifest.name, args.name))
return 1
help(getattr(module.namespace, args.name))
return 0
assert False, "unhandled mode: {}".format(self.mode)
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug("reading configuration file:", filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError('file "{}" included by "{}" does not exist'.format(str(exc), filename))
continue
elif section == "__global__":
prefix = ""
else:
prefix = section + "."
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result
def download_file(url,
filename=None,
file=None,
directory=None,
on_exists='rename',
progress=None,
chunksize=4096,
urlopen_kwargs=None):
"""
Download a file from a URL to one of the following destinations:
:param filename: A filename to write the downloaded file to.
:param file: A file-like object.
:param directory: A directory. The filename will be automatically
determined from the ``Content-Disposition`` header received from
the server or the last path elemnet in the URL.
Additional parameters for the *directory* parameter:
:param on_exists: The operation to perform when the file already exists.
Available modes are ``rename``, ``overwrite`` and ``skip``.
Additional parameters:
:param progress: A callable that accepts a single parameter that is a
dictionary with information about the progress of the download. The
dictionary provides the keys ``size``, ``downloaded`` and ``response``.
If the callable returns :const:`False` (specifically the value False), the
download will be aborted and a :class:`UserInterrupt` will be raised.
:param urlopen_kwargs: A dictionary with additional keyword arguments
for :func:`urllib.request.urlopen`.
Raise and return:
:raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
:raise URLError: Can be raised by :func:`urllib.request.urlopen`.
:raise UserInterrupt: If the *progress* returned :const:`False`.
:return: If the download mode is *directory*, the name of the downloaded
file will be returned and False if the file was newly downloaded, True
if the download was skipped because the file already existed.
Otherwise, the number of bytes downloaded will be returned.
"""
argspec.validate('on_exists', on_exists,
{'enum': ['rename', 'overwrite', 'skip']})
if sum(map(bool, [filename, file, directory])) != 1:
raise ValueError(
'exactly one of filename, file or directory must be specifed')
response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))
if directory:
try:
filename = parse_content_disposition(
response.headers.get('Content-Disposition', ''))
except ValueError:
filename = url.split('/')[-1]
filename = path.join(directory, filename)
path.makedirs(directory)
if path.exists(filename):
if on_exists == 'skip':
return filename, True
elif on_exists == 'rename':
index = 0
while True:
new_filename = filename + '_{:0>4}'.format(index)
if not path.exists(new_filename):
filename = new_filename
break
index += 1
elif on_exists != 'overwrite':
raise RuntimeError
try:
size = int(response.headers.get('Content-Length', ''))
except ValueError:
size = None
progress_info = {
'response': response,
'size': size,
'downloaded': 0,
'completed': False,
'filename': filename,
'url': url
}
if progress and progress(progress_info) is False:
raise UserInterrupt
def copy_to_file(fp):
while True:
data = response.read(chunksize)
if not data:
break
progress_info['downloaded'] += len(data)
fp.write(data)
if progress and progress(progress_info) is False:
raise UserInterrupt
progress_info['completed'] = True
if progress and progress(progress_info) is False:
raise UserInterrupt
if filename:
path.makedirs(path.dirname(filename))
try:
with open(filename, 'wb') as fp:
copy_to_file(fp)
except BaseException:
# Delete the file if it could not be downloaded successfully.
path.remove(filename, silent=True)
raise
elif file:
copy_to_file(file)
return filename, False
def download_file(url, filename=None, file=None, directory=None,
on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None):
"""
Download a file from a URL to one of the following destinations:
:param filename: A filename to write the downloaded file to.
:param file: A file-like object.
:param directory: A directory. The filename will be automatically
determined from the ``Content-Disposition`` header received from
the server or the last path elemnet in the URL.
Additional parameters for the *directory* parameter:
:param on_exists: The operation to perform when the file already exists.
Available modes are ``rename``, ``overwrite`` and ``skip``.
Additional parameters:
:param progress: A callable that accepts a single parameter that is a
dictionary with information about the progress of the download. The
dictionary provides the keys ``size``, ``downloaded`` and ``response``.
If the callable returns :const:`False` (specifically the value False), the
download will be aborted and a :class:`UserInterrupt` will be raised.
:param urlopen_kwargs: A dictionary with additional keyword arguments
for :func:`urllib.request.urlopen`.
Raise and return:
:raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
:raise URLError: Can be raised by :func:`urllib.request.urlopen`.
:raise UserInterrupt: If the *progress* returned :const:`False`.
:return: If the download mode is *directory*, the name of the downloaded
file will be returned and False if the file was newly downloaded, True
if the download was skipped because the file already existed.
Otherwise, the number of bytes downloaded will be returned.
"""
argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']})
if sum(map(bool, [filename, file, directory])) != 1:
raise ValueError('exactly one of filename, file or directory must be specifed')
response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))
if directory:
try:
filename = parse_content_disposition(
response.headers.get('Content-Disposition', ''))
except ValueError:
filename = url.split('/')[-1]
filename = path.join(directory, filename)
path.makedirs(directory)
if path.exists(filename):
if on_exists == 'skip':
return filename, True
elif on_exists == 'rename':
index = 0
while True:
new_filename = filename + '_{:0>4}'.format(index)
if not path.exists(new_filename):
filename = new_filename
break
index += 1
elif on_exists != 'overwrite':
raise RuntimeError
try:
size = int(response.headers.get('Content-Length', ''))
except ValueError:
size = None
progress_info = {'response': response, 'size': size, 'downloaded': 0,
'completed': False, 'filename': filename, 'url': url}
if progress and progress(progress_info) is False:
raise UserInterrupt
def copy_to_file(fp):
while True:
data = response.read(chunksize)
if not data:
break
progress_info['downloaded'] += len(data)
fp.write(data)
if progress and progress(progress_info) is False:
raise UserInterrupt
progress_info['completed'] = True
if progress and progress(progress_info) is False:
raise UserInterrupt
if filename:
path.makedirs(path.dirname(filename))
try:
with open(filename, 'wb') as fp:
copy_to_file(fp)
except BaseException:
# Delete the file if it could not be downloaded successfully.
path.remove(filename, silent=True)
raise
elif file:
copy_to_file(file)
return filename, False
def read_config_file(filename, basedir=None, follow_include_directives=True):
"""
Reads a configuration file and returns a dictionary of the values that
it contains. The format is standard :mod:`configparser` ``.ini`` style,
however this function supports ``include`` directives that can include
additional configuration files.
::
[include "path/to/config.ini"] ; errors if the file does not exist
[include "path/to/config.ini" if-exists] ; ignored if the file does not exist
:param filename: The name of the configuration file to read.
:param basedir: If *filename* is not an absolute path or the base directory
should be altered, this is the directory of which to look for files
specified with ``include`` directives.
:param follow_include_directives: If this is True, ``include`` directives
will be followed.
:raise FileNotFoundError: If *filename* does not exist.
:raise InvalidConfigError: If the configuration format is invalid. Also
if any of the included files do not exist.
:return: A dictionary. Section names are prepended to the option names.
"""
filename = path.norm(filename)
if not basedir:
basedir = path.dirname(filename)
if not path.isfile(filename):
raise FileNotFoundError(filename)
logger.debug('reading configuration file:', filename)
parser = configparser.SafeConfigParser()
try:
parser.read([filename])
except configparser.Error as exc:
raise InvalidConfigError('"{}": {}'.format(filename, exc))
result = {}
for section in parser.sections():
match = re.match('include\s+"([^"]+)"(\s+if-exists)?$', section)
if match:
if not follow_include_directives:
continue
ifile, if_exists = match.groups()
ifile = path.norm(ifile, basedir)
try:
result.update(read_config_file(ifile))
except FileNotFoundError as exc:
if not if_exists:
raise InvalidConfigError(
'file "{}" included by "{}" does not exist'.format(
str(exc), filename))
continue
elif section == '__global__':
prefix = ''
else:
prefix = section + '.'
for option in parser.options(section):
result[prefix + option] = parser.get(section, option)
return result