def write_command_file(self, filename, commands, inputs=None, outputs=None,
cwd=None, environ=None, foreach=False, suffix='.cmd', dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = ['cmd', '/Q', '/c', filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(path.abs(filename)))
with open(filename, 'w') as fp:
fp.write('REM This file is automatically generated with Craftr. It is \n')
fp.write('REM not recommended to modify it manually.\n\n')
if cwd is not None:
fp.write('cd ' + shell.quote(cwd) + '\n\n')
for key, value in environ.items():
fp.write('set ' + shell.quote('{}={}'.format(key, value), for_ninja=True) + '\n')
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands)-1:
command.append(shell.safe('%*'))
fp.write(shell.join(command) + '\n')
fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n')
return result, filename
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, 'craftr')
path.makedirs(directory)
mfile = path.join(directory, 'manifest.' + args.format)
sfile = path.join(directory, 'Craftrfile')
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, 'w') as fp:
if args.format == 'cson':
lines = textwrap.dedent('''
name: "%s"
version: "%s"
project_dir: ".."
author: ""
url: ""
dependencies: {}
options: {}
''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[2]
elif args.format == 'json':
lines = textwrap.dedent('''
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}''' % (args.name, args.version)).lstrip().split('\n')
if not args.nested:
del lines[3]
fp.write('\n'.join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, 'w') as fp:
print('# {}'.format(args.name), file=fp)
def write_cache(cachefile):
# Write back the cache.
try:
path.makedirs(path.dirname(cachefile))
with open(cachefile, 'w') as fp:
session.write_cache(fp)
except OSError as exc:
logger.error('error writing cache file:', cachefile)
logger.error(exc, indent=1)
else:
logger.debug('cache written:', cachefile)
def write_cache(cachefile):
# Write back the cache.
try:
path.makedirs(path.dirname(cachefile))
with open(cachefile, 'w') as fp:
session.write_cache(fp)
except OSError as exc:
logger.error('error writing cache file:', cachefile)
logger.error(exc, indent=1)
else:
logger.debug('cache written:', cachefile)
def execute(self, parser, args):
directory = args.directory or args.name
if path.maybedir(directory):
directory = path.join(directory, args.name)
if not path.exists(directory):
logger.debug('creating directory "{}"'.format(directory))
path.makedirs(directory)
elif not path.isdir(directory):
logger.error('"{}" is not a directory'.format(directory))
return 1
if args.nested:
directory = path.join(directory, "craftr")
path.makedirs(directory)
mfile = path.join(directory, MANIFEST_FILENAME)
sfile = path.join(directory, "Craftrfile")
for fn in [mfile, sfile]:
if path.isfile(fn):
logger.error('"{}" already exists'.format(fn))
return 1
logger.debug('creating file "{}"'.format(mfile))
with open(mfile, "w") as fp:
lines = (
textwrap.dedent(
"""
{
"name": "%s",
"version": "%s",
"project_dir": "..",
"author": "",
"url": "",
"dependencies": {},
"options": {}
}\n"""
% (args.name, args.version)
)
.lstrip()
.split("\n")
)
if not args.nested:
del lines[3]
fp.write("\n".join(lines))
logger.debug('creating file "{}"'.format(sfile))
with open(sfile, "w") as fp:
print("# {}".format(args.name), file=fp)
def write_command_file(self,
filename,
commands,
inputs=None,
outputs=None,
cwd=None,
environ=None,
foreach=False,
suffix='.sh',
dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = [filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
# TODO: Make sure this also works for shells other than bash.
fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) +
'\n')
fp.write('set -e\n')
if cwd:
fp.write('cd ' + shell.quote(cwd) + '\n')
fp.write('\n')
for key, value in environ.items():
fp.write('export {}={}\n'.format(key, shell.quote(value)))
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands) - 1:
command.append(shell.safe('$*'))
fp.write(shell.join(command))
fp.write('\n')
os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # rwxrw-r--
return result, filename
def execute(self, parser, args):
if hasattr(args, 'include_path'):
session.path.extend(map(path.norm, args.include_path))
# Help-command preprocessing. Check if we're to show the help on a builtin
# object, otherwise extract the module name if applicable.
if self.mode == 'help':
if not args.name:
help('craftr')
return 0
if args.name in vars(craftr.defaults):
help(getattr(craftr.defaults, args.name))
return 0
# Check if we have an absolute symbol reference.
if ':' in args.name:
if args.module:
parser.error(
'-m/--module option conflicting with name argument: "{}"'
.format(args.name))
args.module, args.name = args.name.split(':', 1)
module = self._find_module(parser, args)
session.main_module = module
self.ninja_bin, self.ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(path.norm(args.build_dir, INIT_DIR))
path.makedirs(session.builddir)
os.chdir(session.builddir)
self.cachefile = path.join(session.builddir, '.craftrcache')
# Prepare options, loaders and execute.
if self.mode in ('export', 'run', 'help'):
return self._export_run_or_help(args, module)
elif self.mode == 'dump-options':
return self._dump_options(args, module)
elif self.mode == 'dump-deptree':
return self._dump_deptree(args, module)
elif self.mode in ('build', 'clean'):
return self._build_or_clean(args)
elif self.mode == 'lock':
self._create_lockfile()
else:
raise RuntimeError("mode: {}".format(self.mode))
def write_command_file(self,
filename,
commands,
inputs=None,
outputs=None,
cwd=None,
environ=None,
foreach=False,
suffix='.cmd',
dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = ['cmd', '/Q', '/c', filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(path.abs(filename)))
with open(filename, 'w') as fp:
fp.write(
'REM This file is automatically generated with Craftr. It is \n'
)
fp.write('REM not recommended to modify it manually.\n\n')
if cwd is not None:
fp.write('cd ' + shell.quote(cwd) + '\n\n')
for key, value in environ.items():
fp.write(
'set ' +
shell.quote('{}={}'.format(key, value), for_ninja=True) +
'\n')
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands) - 1:
command.append(shell.safe('%*'))
fp.write(shell.join(command) + '\n')
fp.write('if %errorlevel% neq 0 exit %errorlevel%\n\n')
return result, filename
def write_response_file(arguments, builder=None, name=None, force_file=False, suffix=''):
"""
Creates a response-file with the specified *name* in the in the
``buildfiles/`` directory and writes the *arguments* list quoted into
the file. If *builder* is specified, it must be a :class:`TargetBuilder`
and the response file will be added to the implicit dependencies.
If *force_file* is set to True, a file will always be written. Otherwise,
the function will into possible limitations of the platform and decide
whether to write a response file or to return the *arguments* as is.
Returns a tuple of ``(filename, arguments)``. If a response file is written,
the returned *arguments* will be a list with a single string that is the
filename prepended with ``@``. The *filename* part can be None if no
response file needed to be exported.
"""
if not name:
if not builder:
raise ValueError('builder must be specified if name is bot')
name = builder.name + suffix + '.response.txt'
if platform.name != 'win':
return None, arguments
# We'll just assume that there won't be more than 2048 characters for
# other flags. The windows max buffer size is 8192.
content = shell.join(arguments)
if len(content) < 6144:
return None, arguments
filename = buildlocal(path.join('buildfiles', name))
if builder:
builder.implicit_deps.append(filename)
if session.builddir:
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
fp.write(content)
return filename, ['@' + filename]
def write_response_file(arguments, builder=None, name=None, force_file=False):
"""
Creates a response-file with the specified *name* in the in the
``buildfiles/`` directory and writes the *arguments* list quoted into
the file. If *builder* is specified, it must be a :class:`TargetBuilder`
and the response file will be added to the implicit dependencies.
If *force_file* is set to True, a file will always be written. Otherwise,
the function will into possible limitations of the platform and decide
whether to write a response file or to return the *arguments* as is.
Returns a tuple of ``(filename, arguments)``. If a response file is written,
the returned *arguments* will be a list with a single string that is the
filename prepended with ``@``. The *filename* part can be None if no
response file needed to be exported.
"""
if not name:
if not builder:
raise ValueError('builder must be specified if name is bot')
name = builder.name + '.response.txt'
if platform.name != 'win':
return None, arguments
# We'll just assume that there won't be more than 2048 characters for
# other flags. The windows max buffer size is 8192.
content = shell.join(arguments)
if len(content) < 6144:
return None, arguments
filename = buildlocal(path.join('buildfiles', name))
if builder:
builder.implicit_deps.append(filename)
if session.builddir:
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
fp.write(content)
return filename, ['@' + filename]
def write_command_file(self, filename, commands, inputs=None, outputs=None,
cwd=None, environ=None, foreach=False, suffix='.sh', dry=False,
accept_additional_args=False):
if suffix is not None:
filename = path.addsuffix(filename, suffix)
result = [filename]
if foreach:
result += ['$in', '$out']
inputs, outputs = ['%1'], ['%2']
commands = self.replace_commands_inout_vars(commands, inputs, outputs)
if dry:
return result, filename
path.makedirs(path.dirname(filename))
with open(filename, 'w') as fp:
# TODO: Make sure this also works for shells other than bash.
fp.write('#!' + shell.find_program(environ.get('SHELL', 'bash')) + '\n')
fp.write('set -e\n')
if cwd:
fp.write('cd ' + shell.quote(cwd) + '\n')
fp.write('\n')
for key, value in environ.items():
fp.write('export {}={}\n'.format(key, shell.quote(value)))
fp.write('\n')
for index, command in enumerate(commands):
if accept_additional_args and index == len(commands)-1:
command.append(shell.safe('$*'))
fp.write(shell.join(command))
fp.write('\n')
os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # rwxrw-r--
return result, filename
def download_file(url,
filename=None,
file=None,
directory=None,
on_exists='rename',
progress=None,
chunksize=4096,
urlopen_kwargs=None):
"""
Download a file from a URL to one of the following destinations:
:param filename: A filename to write the downloaded file to.
:param file: A file-like object.
:param directory: A directory. The filename will be automatically
determined from the ``Content-Disposition`` header received from
the server or the last path elemnet in the URL.
Additional parameters for the *directory* parameter:
:param on_exists: The operation to perform when the file already exists.
Available modes are ``rename``, ``overwrite`` and ``skip``.
Additional parameters:
:param progress: A callable that accepts a single parameter that is a
dictionary with information about the progress of the download. The
dictionary provides the keys ``size``, ``downloaded`` and ``response``.
If the callable returns :const:`False` (specifically the value False), the
download will be aborted and a :class:`UserInterrupt` will be raised.
:param urlopen_kwargs: A dictionary with additional keyword arguments
for :func:`urllib.request.urlopen`.
Raise and return:
:raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
:raise URLError: Can be raised by :func:`urllib.request.urlopen`.
:raise UserInterrupt: If the *progress* returned :const:`False`.
:return: If the download mode is *directory*, the name of the downloaded
file will be returned and False if the file was newly downloaded, True
if the download was skipped because the file already existed.
Otherwise, the number of bytes downloaded will be returned.
"""
argspec.validate('on_exists', on_exists,
{'enum': ['rename', 'overwrite', 'skip']})
if sum(map(bool, [filename, file, directory])) != 1:
raise ValueError(
'exactly one of filename, file or directory must be specifed')
response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))
if directory:
try:
filename = parse_content_disposition(
response.headers.get('Content-Disposition', ''))
except ValueError:
filename = url.split('/')[-1]
filename = path.join(directory, filename)
path.makedirs(directory)
if path.exists(filename):
if on_exists == 'skip':
return filename, True
elif on_exists == 'rename':
index = 0
while True:
new_filename = filename + '_{:0>4}'.format(index)
if not path.exists(new_filename):
filename = new_filename
break
index += 1
elif on_exists != 'overwrite':
raise RuntimeError
try:
size = int(response.headers.get('Content-Length', ''))
except ValueError:
size = None
progress_info = {
'response': response,
'size': size,
'downloaded': 0,
'completed': False,
'filename': filename,
'url': url
}
if progress and progress(progress_info) is False:
raise UserInterrupt
def copy_to_file(fp):
while True:
data = response.read(chunksize)
if not data:
break
progress_info['downloaded'] += len(data)
fp.write(data)
if progress and progress(progress_info) is False:
raise UserInterrupt
progress_info['completed'] = True
if progress and progress(progress_info) is False:
raise UserInterrupt
if filename:
path.makedirs(path.dirname(filename))
try:
with open(filename, 'wb') as fp:
copy_to_file(fp)
except BaseException:
# Delete the file if it could not be downloaded successfully.
path.remove(filename, silent=True)
raise
elif file:
copy_to_file(file)
return filename, False
def download_file(url, filename=None, file=None, directory=None,
on_exists='rename', progress=None, chunksize=4096, urlopen_kwargs=None):
"""
Download a file from a URL to one of the following destinations:
:param filename: A filename to write the downloaded file to.
:param file: A file-like object.
:param directory: A directory. The filename will be automatically
determined from the ``Content-Disposition`` header received from
the server or the last path elemnet in the URL.
Additional parameters for the *directory* parameter:
:param on_exists: The operation to perform when the file already exists.
Available modes are ``rename``, ``overwrite`` and ``skip``.
Additional parameters:
:param progress: A callable that accepts a single parameter that is a
dictionary with information about the progress of the download. The
dictionary provides the keys ``size``, ``downloaded`` and ``response``.
If the callable returns :const:`False` (specifically the value False), the
download will be aborted and a :class:`UserInterrupt` will be raised.
:param urlopen_kwargs: A dictionary with additional keyword arguments
for :func:`urllib.request.urlopen`.
Raise and return:
:raise HTTPError: Can be raised by :func:`urllib.request.urlopen`.
:raise URLError: Can be raised by :func:`urllib.request.urlopen`.
:raise UserInterrupt: If the *progress* returned :const:`False`.
:return: If the download mode is *directory*, the name of the downloaded
file will be returned and False if the file was newly downloaded, True
if the download was skipped because the file already existed.
Otherwise, the number of bytes downloaded will be returned.
"""
argspec.validate('on_exists', on_exists, {'enum': ['rename', 'overwrite', 'skip']})
if sum(map(bool, [filename, file, directory])) != 1:
raise ValueError('exactly one of filename, file or directory must be specifed')
response = urllib.request.urlopen(url, **(urlopen_kwargs or {}))
if directory:
try:
filename = parse_content_disposition(
response.headers.get('Content-Disposition', ''))
except ValueError:
filename = url.split('/')[-1]
filename = path.join(directory, filename)
path.makedirs(directory)
if path.exists(filename):
if on_exists == 'skip':
return filename, True
elif on_exists == 'rename':
index = 0
while True:
new_filename = filename + '_{:0>4}'.format(index)
if not path.exists(new_filename):
filename = new_filename
break
index += 1
elif on_exists != 'overwrite':
raise RuntimeError
try:
size = int(response.headers.get('Content-Length', ''))
except ValueError:
size = None
progress_info = {'response': response, 'size': size, 'downloaded': 0,
'completed': False, 'filename': filename, 'url': url}
if progress and progress(progress_info) is False:
raise UserInterrupt
def copy_to_file(fp):
while True:
data = response.read(chunksize)
if not data:
break
progress_info['downloaded'] += len(data)
fp.write(data)
if progress and progress(progress_info) is False:
raise UserInterrupt
progress_info['completed'] = True
if progress and progress(progress_info) is False:
raise UserInterrupt
if filename:
path.makedirs(path.dirname(filename))
try:
with open(filename, 'wb') as fp:
copy_to_file(fp)
except BaseException:
# Delete the file if it could not be downloaded successfully.
path.remove(filename, silent=True)
raise
elif file:
copy_to_file(file)
return filename, False
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.is_export:
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join('craftr', MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error('{} (note: you have to escape > and < characters)'.format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error('module not found: ' + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, '.craftrcache')
if not read_cache(cachefile) and not self.is_export:
logger.error('Unable to load "{}", can not build'.format(cachefile))
return 1
# Prepare options, loaders and execute.
if self.is_export:
session.cache['build'] = {}
try:
write_cache(cachefile)
module.run()
except (Module.InvalidOption, Module.LoaderInitializationError) as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error(exc)
return 1
# Write the cache back.
session.cache['build']['targets'] = list(session.graph.targets.keys())
session.cache['build']['main'] = module.ident
session.cache['build']['options'] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", 'w') as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache['build']['options'])
main = session.cache['build']['main']
available_targets = frozenset(session.cache['build']['targets'])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if '.' not in target:
target = main + '.' + target
elif target.startswith('.'):
target = main + target
module_name, target = target.rpartition('.')[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or '*')
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error('no such target: {}'.format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ['-v']
cmd += targets
shell.run(cmd)
def execute(self, parser, args):
session.path.extend(map(path.norm, args.include_path))
if self.mode == "export":
# Determine the module to execute, either from the current working
# directory or find it by name if one is specified.
if not args.module:
for fn in [MANIFEST_FILENAME, path.join("craftr", MANIFEST_FILENAME)]:
if path.isfile(fn):
module = session.parse_manifest(fn)
break
else:
parser.error('"{}" does not exist'.format(MANIFEST_FILENAME))
else:
# TODO: For some reason, prints to stdout are not visible here.
# TODO: Prints to stderr however work fine.
try:
module_name, version = parse_module_spec(args.module)
except ValueError as exc:
parser.error("{} (note: you have to escape > and < characters)".format(exc))
try:
module = session.find_module(module_name, version)
except Module.NotFound as exc:
parser.error("module not found: " + str(exc))
else:
module = None
ninja_bin, ninja_version = get_ninja_info()
# Create and switch to the build directory.
session.builddir = path.abs(args.build_dir)
path.makedirs(session.builddir)
os.chdir(session.builddir)
# Read the cache and parse command-line options.
cachefile = path.join(session.builddir, ".craftrcache")
if not read_cache(cachefile) and self.mode != "export":
logger.error('Unable to load "{}", can not {}'.format(cachefile, self.mode))
logger.error("Make sure to generate a build tree with 'craftr export'")
return 1
# Prepare options, loaders and execute.
if self.mode == "export":
session.expand_relative_options(module.manifest.name)
session.cache["build"] = {}
try:
module.run()
except Module.InvalidOption as exc:
for error in exc.format_errors():
logger.error(error)
return 1
except craftr.defaults.ModuleError as exc:
logger.error("error:", exc)
return 1
finally:
if sys.exc_info():
# We still want to write the cache, especially so that data already
# loaded with loaders doesn't need to be re-loaded. They'll find out
# when the cached information was not valid.
write_cache(cachefile)
# Write the cache back.
session.cache["build"]["targets"] = list(session.graph.targets.keys())
session.cache["build"]["main"] = module.ident
session.cache["build"]["options"] = args.options
write_cache(cachefile)
# Write the Ninja manifest.
with open("build.ninja", "w") as fp:
platform = core.build.get_platform_helper()
context = core.build.ExportContext(ninja_version)
writer = core.build.NinjaWriter(fp)
session.graph.export(writer, context, platform)
else:
parse_cmdline_options(session.cache["build"]["options"])
main = session.cache["build"]["main"]
available_targets = frozenset(session.cache["build"]["targets"])
logger.debug("build main module:", main)
session.expand_relative_options(get_volatile_module_version(main)[0])
# Check the targets and if they exist.
targets = []
for target in args.targets:
if "." not in target:
target = main + "." + target
elif target.startswith("."):
target = main + target
module_name, target = target.rpartition(".")[::2]
module_name, version = get_volatile_module_version(module_name)
ref_module = session.find_module(module_name, version or "*")
target = craftr.targetbuilder.get_full_name(target, ref_module)
if target not in available_targets:
parser.error("no such target: {}".format(target))
targets.append(target)
# Execute the ninja build.
cmd = [ninja_bin]
if args.verbose:
cmd += ["-v"]
if self.mode == "clean":
cmd += ["-t", "clean"]
if not args.recursive:
cmd += ["-r"]
cmd += targets
return shell.run(cmd).returncode
