def terminal(port):
click.echo(click.style('NOTE: This is an early prototype of the terminal.'
' Nothing is guaranteed to work.', bold=True))
if port == 'default':
if len(prosflasher.ports.list_com_ports()) == 1:
port = prosflasher.ports.list_com_ports()[0].device
elif len(prosflasher.ports.list_com_ports()) > 1:
click.echo('Multiple ports were found:')
click.echo(prosflasher.ports.create_port_list())
port = click.prompt('Select a port to open',
type=click.Choice([p.device for p in prosflasher.ports.list_com_ports()]))
else:
click.echo('No ports were found.')
click.get_current_context().abort()
sys.exit()
ser = prosflasher.ports.create_serial(port)
term = proscli.serial_terminal.Terminal(ser)
signal.signal(signal.SIGINT, term.stop)
term.start()
while term.alive:
time.sleep(0.005)
term.join()
ser.close()
print('Exited successfully')
sys.exit(0)
def resolve_id_or_name(client, bookmark_id_or_name):
# leading/trailing whitespace doesn't make sense for UUIDs and the Transfer
# service outright forbids it for bookmark names, so we can strip it off
bookmark_id_or_name = bookmark_id_or_name.strip()
res = None
try:
UUID(bookmark_id_or_name) # raises ValueError if argument not a UUID
except ValueError:
pass
else:
try:
res = client.get_bookmark(bookmark_id_or_name.lower())
except TransferAPIError as exception:
if exception.code != "BookmarkNotFound":
raise
if not res: # non-UUID input or UUID not found; fallback to match by name
try:
# n.b. case matters to the Transfer service for bookmark names, so
# two bookmarks can exist whose names vary only by their case
res = next(
bookmark_row
for bookmark_row in client.bookmark_list()
if bookmark_row["name"] == bookmark_id_or_name
)
except StopIteration:
safeprint(
u'No bookmark found for "{}"'.format(bookmark_id_or_name),
write_to_stderr=True,
)
click.get_current_context().exit(1)
return res
def upgrade(cfg, kernel, location, depot):
first_run(cfg)
templates = local.get_local_templates(pros_cfg=cfg.pros_cfg,
template_types=[TemplateTypes.kernel]) # type: List[Identifier]
if not templates or len(templates) == 0:
click.echo('No templates have been downloaded! Use `pros conduct download` to download the latest kernel.')
click.get_current_context().abort()
sys.exit()
kernel_version = kernel
if kernel == 'latest':
kernel_version = sorted(templates, key=lambda t: semver.Version(t.version))[-1].version
proscli.utils.debug('Resolved version {} to {}'.format(kernel, kernel_version))
templates = [t for t in templates if t.version == kernel_version]
depot_registrar = depot
if depot == 'auto':
templates = [t for t in templates if t.version == kernel_version]
if not templates or len(templates) == 0:
click.echo('No templates exist for {}'.format(kernel_version))
click.get_current_context().abort()
sys.exit()
if 'pros-mainline' in [t.depot for t in templates]:
depot_registrar = 'pros-mainline'
else:
depot_registrar = [t.depot for t in templates][0]
proscli.utils.debug('Resolved depot {} to {}'.format(depot, depot_registrar))
templates = [t for t in templates if t.depot == depot_registrar]
if not templates or len(templates) == 0:
click.echo('No templates were found for kernel version {} on {}'.format(kernel_version, depot_registrar))
template = templates[0]
if not os.path.isabs(location):
location = os.path.abspath(location)
click.echo('Upgrading existing project from {} on {} at {}'.format(template.version, template.depot, location))
local.upgrade_project(identifier=template, dest=location, pros_cli=cfg.pros_cfg)
def handle_in_params(self, kwargs):
import q2cli.handlers
arguments = {}
missing = []
cmd_fallback = self.cmd_config_handler.get_value(kwargs)
verbose = self.verbose_handler.get_value(kwargs, fallback=cmd_fallback)
quiet = self.quiet_handler.get_value(kwargs, fallback=cmd_fallback)
if verbose and quiet:
click.secho('Unsure of how to be quiet and verbose at the '
'same time.', fg='red', bold=True, err=True)
click.get_current_context().exit(1)
for item in self.action['signature']:
if item['type'] == 'input' or item['type'] == 'parameter':
name = item['name']
handler = self.generated_handlers[name]
try:
if isinstance(handler,
(q2cli.handlers.MetadataHandler,
q2cli.handlers.MetadataColumnHandler)):
arguments[name] = handler.get_value(
verbose, kwargs, fallback=cmd_fallback)
else:
arguments[name] = handler.get_value(
kwargs, fallback=cmd_fallback)
except q2cli.handlers.ValueNotFoundException:
missing += handler.missing
return arguments, missing, verbose, quiet
def autoactivate(client, endpoint_id, if_expires_in=None):
"""
Attempts to auto-activate the given endpoint with the given client
If auto-activation fails, parses the returned activation requirements
to determine which methods of activation are supported, then tells
the user to use 'globus endpoint activate' with the correct options(s)
"""
kwargs = {}
if if_expires_in is not None:
kwargs["if_expires_in"] = if_expires_in
res = client.endpoint_autoactivate(endpoint_id, **kwargs)
if res["code"] == "AutoActivationFailed":
message = (
"The endpoint could not be auto-activated and must be "
"activated before it can be used.\n\n"
+ activation_requirements_help_text(res, endpoint_id)
)
safeprint(message, write_to_stderr=True)
click.get_current_context().exit(1)
else:
return res
def create_project(identifier, dest, pros_cli=None):
if pros_cli is None or not pros_cli:
pros_cli = CliConfig()
filename = os.path.join(pros_cli.directory, identifier.depot,
'{}-{}'.format(identifier.name, identifier.version),
'template.pros')
if not os.path.isfile(filename):
click.echo('Error: template.pros not found for {}-{}'.format(identifier.name, identifier.version))
click.get_current_context().abort()
sys.exit()
if os.path.isfile(dest) or (os.path.isdir(dest) and len(os.listdir(dest)) > 0):
click.echo('Error! Destination is a file or a nonempty directory! Delete the file(s) and try again.')
click.get_current_context().abort()
sys.exit()
config = TemplateConfig(file=filename)
distutils.dir_util.copy_tree(config.directory, dest)
for root, dirs, files in os.walk(dest):
for d in dirs:
if any([fnmatch.fnmatch(d, p) for p in config.template_ignore]):
verbose('Removing {}'.format(d))
os.rmdir(os.path.join(root, d))
for f in files:
if any([fnmatch.fnmatch(f, p) for p in config.template_ignore]):
verbose('Removing {}'.format(f))
os.remove(os.path.join(root, f))
proj_config = prosconfig.ProjectConfig(dest, create=True)
proj_config.kernel = identifier.version
proj_config.save()
def check_completed():
completed = client.task_wait(
task_id, timeout=polling_interval, polling_interval=polling_interval
)
if completed:
if heartbeat:
safeprint("", write_to_stderr=True)
# meowing tasks wake up!
if meow:
safeprint(
r"""
_..
/}_{\ /.-'
( a a )-.___...-'/
==._.== ;
\ i _..._ /,
{_;/ {_//""",
write_to_stderr=True,
)
# TODO: possibly update TransferClient.task_wait so that we don't
# need to do an extra fetch to get the task status after completion
res = client.get_task(task_id)
formatted_print(res, text_format=FORMAT_SILENT)
status = res["status"]
if status == "SUCCEEDED":
click.get_current_context().exit(0)
else:
click.get_current_context().exit(1)
return completed
def upgrade_project(identifier, dest, pros_cli=None):
if pros_cli is None or not pros_cli:
pros_cli = CliConfig()
filename = os.path.join(pros_cli.directory, identifier.depot,
'{}-{}'.format(identifier.name, identifier.version),
'template.pros')
if not os.path.isfile(filename):
click.echo('Error: template.pros not found for {}-{}'.format(identifier.name, identifier.version))
click.get_current_context().abort()
sys.exit()
proj_config = prosconfig.ProjectConfig(dest, raise_on_error=True)
config = TemplateConfig(file=filename)
for root, dirs, files in os.walk(config.directory):
for d in dirs:
if any([fnmatch.fnmatch(d, p) for p in config.upgrade_paths]):
verbose('Upgrading {}'.format(d))
relpath = os.path.relpath(os.path.join(root, d), config.directory)
shutil.copytree(os.path.join(config.directory, relpath), os.path.join(proj_config.directory, relpath))
for f in files:
if any([fnmatch.fnmatch(f, p) for p in config.upgrade_paths]):
verbose('Upgrading {}'.format(f))
relpath = os.path.relpath(os.path.join(root, f), config.directory)
shutil.copyfile(os.path.join(config.directory, relpath), os.path.join(proj_config.directory, relpath))
def exit_with_error(e, header='An error has been encountered:', file=None,
suppress_footer=False):
import sys
import traceback
import textwrap
import click
if file is None:
file = sys.stderr
footer = 'See above for debug info.'
else:
footer = 'Debug info has been saved to %s' % file.name
error = textwrap.indent(str(e), ' ')
segments = [header, error]
if not suppress_footer:
segments.append(footer)
traceback.print_exception(type(e), e, e.__traceback__, file=file)
file.write('\n')
click.secho('\n\n'.join(segments), fg='red', bold=True, err=True)
click.get_current_context().exit(1)
def exit_with_error(e, header='An error has been encountered:',
traceback='stderr', status=1):
import sys
import traceback as tb
import textwrap
import click
footer = [] # footer only exists if traceback is set
tb_file = None
if traceback == 'stderr':
tb_file = sys.stderr
footer = ['See above for debug info.']
elif traceback is not None:
tb_file = traceback
footer = ['Debug info has been saved to %s' % tb_file.name]
error = textwrap.indent(str(e), ' ')
segments = [header, error] + footer
if traceback is not None:
tb.print_exception(type(e), e, e.__traceback__, file=tb_file)
tb_file.write('\n')
click.secho('\n\n'.join(segments), fg='red', bold=True, err=True)
if not footer:
click.echo(err=True) # extra newline to look normal
click.get_current_context().exit(status)
def whoami_command(linked_identities):
"""
Executor for `globus whoami`
"""
client = get_auth_client()
# get userinfo from auth.
# if we get back an error the user likely needs to log in again
try:
res = client.oauth2_userinfo()
except AuthAPIError:
safeprint(
"Unable to get user information. Please try " "logging in again.",
write_to_stderr=True,
)
click.get_current_context().exit(1)
print_command_hint(
"For information on which identities are in session see\n"
" globus session show\n"
)
# --linked-identities either displays all usernames or a table if verbose
if linked_identities:
try:
formatted_print(
res["identity_set"],
fields=[
("Username", "username"),
("Name", "name"),
("ID", "sub"),
("Email", "email"),
],
simple_text=(
None
if is_verbose()
else "\n".join([x["username"] for x in res["identity_set"]])
),
)
except KeyError:
safeprint(
"Your current login does not have the consents required "
"to view your full identity set. Please log in again "
"to agree to the required consents.",
write_to_stderr=True,
)
# Default output is the top level data
else:
formatted_print(
res,
text_format=FORMAT_TEXT_RECORD,
fields=[
("Username", "preferred_username"),
("Name", "name"),
("ID", "sub"),
("Email", "email"),
],
simple_text=(None if is_verbose() else res["preferred_username"]),
)
def export_data(input_path, output_path, output_format):
import qiime2.util
import qiime2.sdk
import distutils
result = qiime2.sdk.Result.load(input_path)
if output_format is None:
if isinstance(result, qiime2.sdk.Artifact):
output_format = result.format.__name__
else:
output_format = 'Visualization'
result.export_data(output_path)
else:
if isinstance(result, qiime2.sdk.Visualization):
error = '--output-format cannot be used with visualizations'
click.secho(error, fg='red', bold=True, err=True)
click.get_current_context().exit(1)
else:
source = result.view(qiime2.sdk.parse_format(output_format))
if os.path.isfile(str(source)):
if os.path.isfile(output_path):
os.remove(output_path)
else:
# create directory (recursively) if it doesn't exist yet
os.makedirs(os.path.dirname(output_path), exist_ok=True)
qiime2.util.duplicate(str(source), output_path)
else:
distutils.dir_util.copy_tree(str(source), output_path)
output_type = 'file' if os.path.isfile(output_path) else 'directory'
success = 'Exported %s as %s to %s %s' % (input_path, output_format,
output_type, output_path)
click.secho(success, fg='green')
def config_depot(cfg, name):
if name not in [d.name for d in utils.get_depot_configs(cfg.pros_cfg)]:
click.echo('{} isn\'t a registered depot! Have you added it using `pros conduct add-depot`?')
click.get_current_context().abort()
sys.exit()
depot = [d for d in utils.get_depot_configs(cfg.pros_cfg) if d.name == name][0]
depot.registrar_options = utils.get_all_provider_types(cfg.pros_cfg)[depot.registrar](None) \
.configure_registrar_options(default=depot.registrar_options)
depot.save()
def cli_main(pid, include_greenlet, debugger, verbose):
'''Print stack of python process.
$ pystack <pid>
'''
try:
print_stack(pid, include_greenlet, debugger, verbose)
except DebuggerNotFound as e:
click.echo('DebuggerNotFound: %s' % e.args[0], err=True)
click.get_current_context().exit(1)
def callback(ctx, param, value):
if not value or ctx.resilient_parsing:
return
if value == "BASH":
safeprint(bash_shell_completer)
elif value == "ZSH":
safeprint(zsh_shell_completer)
else:
raise ValueError("Unsupported shell completion")
click.get_current_context().exit(0)
def fail(deadline=None):
exp_string = ""
if deadline is not None:
exp_string = " or will expire within {} seconds".format(deadline)
message = "The endpoint is not activated{}.\n\n".format(
exp_string
) + activation_requirements_help_text(res, endpoint_id)
formatted_print(res, simple_text=message)
click.get_current_context().exit(1)
def make_output_type(index, config):
source_type = index.products.get_by_name(config['source_type'])
if not source_type:
click.echo("Source DatasetType %s does not exist", config['source_type'])
click.get_current_context().exit(1)
output_type = morph_dataset_type(source_type, config)
_LOG.info('Created DatasetType %s', output_type.name)
output_type = index.products.add(output_type)
return source_type, output_type
def __call__(self, **kwargs):
"""Called when user hits return, **kwargs are Dict[click_names, Obj]"""
import importlib
import itertools
import os
import qiime2.util
arguments, missing_in, verbose = self.handle_in_params(kwargs)
outputs, missing_out = self.handle_out_params(kwargs)
if missing_in or missing_out:
# A new context is generated for a callback, which will result in
# the ctx.command_path duplicating the action, so just use the
# parent so we can print the help *within* a callback.
ctx = click.get_current_context().parent
click.echo(ctx.get_help()+"\n", err=True)
for option in itertools.chain(missing_in, missing_out):
click.secho("Error: Missing option: --%s" % option, err=True,
fg='red', bold=True)
if missing_out:
click.echo(_OUTPUT_OPTION_ERR_MSG, err=True)
ctx.exit(1)
module_path = 'qiime2.plugins.%s.actions' % self.plugin['id']
actions_module = importlib.import_module(module_path)
action = getattr(actions_module, self.action['id'])
stdout = os.devnull
stderr = os.devnull
if verbose:
# `qiime2.util.redirected_stdio` defaults to stdout/stderr when
# supplied `None`.
stdout = None
stderr = None
try:
with qiime2.util.redirected_stdio(stdout=stdout,
stderr=stderr):
results = action(**arguments)
except Exception as e:
if verbose:
import traceback
import sys
traceback.print_exc(file=sys.stderr)
click.echo(err=True)
self._echo_plugin_error(e, 'See above for debug info.')
else:
self._echo_plugin_error(
e, 'Re-run with --verbose to see debug info.')
click.get_current_context().exit(1)
for result, output in zip(results, outputs):
click.secho('Saved %s to: %s' % (result.type,
result.save(output)), fg='green')
def test_global_context_object(runner):
@click.command()
@click.pass_context
def cli(ctx):
assert click.get_current_context() is ctx
ctx.obj = 'FOOBAR'
assert click.get_current_context().obj == 'FOOBAR'
assert click.get_current_context(silent=True) is None
runner.invoke(cli, [], catch_exceptions=False)
assert click.get_current_context(silent=True) is None
def filename_command():
"""
Executor for `globus config filename`
"""
try:
config = get_config_obj(file_error=True)
except IOError as e:
safeprint(e, write_to_stderr=True)
click.get_current_context().exit(1)
else:
safeprint(config.filename)
def list_missing_regions(cli_data):
"""
List missing source sounds required to fully generate a target.
"""
cli_data.populate_vsq()
missing = cli_data.target.missing_vsq_regions()
for region in missing:
click.echo('{0}'.format(region))
if len(missing) > 0:
click.get_current_context().exit(1)
def save(self, file=None):
if file is None:
file = self.save_file
if isinstance(click.get_current_context().obj, proscli.utils.State) and click.get_current_context().obj.debug:
proscli.utils.debug('Pretty Formatting {} File'.format(self.__class__.__name__))
jsonpickle.set_encoder_options('json', sort_keys=True, indent=4)
else:
jsonpickle.set_encoder_options('json', sort_keys=True)
if os.path.dirname(file):
os.makedirs(os.path.dirname(file), exist_ok=True)
with open(file, 'w') as f:
f.write(jsonpickle.encode(self))
def first_run(ctx: proscli.utils.State, force=False, defaults=False):
if len(utils.get_depot_configs(ctx.pros_cfg)) == 0:
click.echo('You don\'t currently have any depots configured.')
if len(utils.get_depot_configs(ctx.pros_cfg)) == 0 or force:
if defaults or click.confirm('Add the official PROS kernel depot, pros-mainline?', default=True):
click.get_current_context().invoke(add_depot, name='pros-mainline',
registrar='github-releases',
location='purduesigbots/pros',
configure=False)
click.echo('Added pros-mainline to available depots. '
'Add more depots in the future by running `pros conduct add-depot`\n')
if defaults or click.confirm('Download the latest kernel?', default=True):
click.get_current_context().invoke(download, name='kernel', depot='pros-mainline')
def upload(port, binary, no_poll=False, ctx=proscli.utils.State()):
if not os.path.isfile(binary):
click.echo('Failed to download... file does not exist')
return False
port = prosflasher.ports.create_serial(port)
if not port:
click.echo('Failed to download: port not found')
return
try:
stop_user_code(port, ctx)
if not no_poll:
sys_info = ask_sys_info(port, ctx)
if sys_info is None:
time.sleep(1.5)
sys_info = ask_sys_info(port)
if sys_info is None:
click.echo('Failed to get system info... Try again', err=True)
click.get_current_context().abort()
sys.exit(1)
click.echo(repr(sys_info))
else:
sys_info = SystemInfo()
sys_info.connection_type = ConnectionType.serial_usb # assume this
if sys_info.connection_type == ConnectionType.unknown:
click.confirm('Unable to determine system type. It may be necessary to press the '
'programming button on the programming kit. Continue?', abort=True, default=True)
if sys_info.connection_type == ConnectionType.serial_vexnet2:
# need to send to download channel
if not send_to_download_channel(port):
return False
if not expose_bootloader(port):
return False
if sys_info.connection_type == ConnectionType.serial_usb:
time.sleep(0.25)
if not prosflasher.bootloader.prepare_bootloader(port):
return False
if not prosflasher.bootloader.erase_flash(port):
return False
if not prosflasher.bootloader.upload_binary(port, binary):
return False
if not prosflasher.bootloader.send_go_command(port, 0x08000000):
return False
reset_cortex(port)
except serial.serialutil.SerialException as e:
click.echo('Failed to download code! ' + str(e))
finally:
port.close()
click.echo("Download complete!")
pass
def validate_filenames(cli_data):
"""
Validates the filenames of target sounds.
"""
have_violations = False
for t in cli_data.target.targets:
violations = t.validate_filenames()
if len(violations) > 0:
have_violations = True
click.echo(t)
for v in violations:
click.echo('{0}{1}'.format(INDENT, v))
if have_violations:
click.get_current_context().exit(1)
def _parse_boolean(self, string):
"""Parse string representing a boolean into Python bool type.
Supported values match `configparser.ConfigParser.getboolean`.
"""
trues = ['1', 'yes', 'true', 'on']
falses = ['0', 'no', 'false', 'off']
string_lower = string.lower()
if string_lower in trues:
return True
elif string_lower in falses:
return False
else:
import itertools
import click
msg = (
"Error: unrecognized value for --%s flag: %s\n"
"Supported values (case-insensitive): %s" %
(self.cli_name, string,
', '.join(itertools.chain(trues, falses)))
)
click.secho(msg, err=True, fg='red', bold=True)
ctx = click.get_current_context()
ctx.exit(1)
def get_code(shell=None, prog_name=None, env_name=None, extra_env=None):
"""Returns the completion code to be evaluated by the shell
Parameters
----------
shell : Shell
The shell type (Default value = None)
prog_name : str
The program name on the command line (Default value = None)
env_name : str
The environment variable used to control the completion (Default value = None)
extra_env : dict
Some extra environment variables to be added to the generated code (Default value = None)
Returns
-------
str
The code to be evaluated by the shell
"""
from jinja2 import Environment, FileSystemLoader
if shell in [None, 'auto']:
shell = get_auto_shell()
if not isinstance(shell, Shell):
shell = Shell[shell]
prog_name = prog_name or click.get_current_context().find_root().info_name
env_name = env_name or '_%s_COMPLETE' % prog_name.upper().replace('-', '_')
extra_env = extra_env if extra_env else {}
env = Environment(loader=FileSystemLoader(os.path.dirname(__file__)))
template = env.get_template('%s.j2' % shell.name)
return template.render(prog_name=prog_name, complete_var=env_name, extra_env=extra_env)
def __call__(self, **kwargs):
"""Called when user hits return, **kwargs are Dict[click_names, Obj]"""
import itertools
import os
import qiime2.util
arguments, missing_in, verbose, quiet = self.handle_in_params(kwargs)
outputs, missing_out = self.handle_out_params(kwargs)
if missing_in or missing_out:
# A new context is generated for a callback, which will result in
# the ctx.command_path duplicating the action, so just use the
# parent so we can print the help *within* a callback.
ctx = click.get_current_context().parent
click.echo(ctx.get_help()+"\n", err=True)
for option in itertools.chain(missing_in, missing_out):
click.secho("Error: Missing option: --%s" % option, err=True,
fg='red', bold=True)
if missing_out:
click.echo(_OUTPUT_OPTION_ERR_MSG, err=True)
ctx.exit(1)
action = self._get_action()
# `qiime2.util.redirected_stdio` defaults to stdout/stderr when
# supplied `None`.
log = None
if not verbose:
import tempfile
log = tempfile.NamedTemporaryFile(prefix='qiime2-q2cli-err-',
suffix='.log',
delete=False, mode='w')
cleanup_logfile = False
try:
with qiime2.util.redirected_stdio(stdout=log, stderr=log):
results = action(**arguments)
except Exception as e:
header = ('Plugin error from %s:'
% q2cli.util.to_cli_name(self.plugin['name']))
if verbose:
# log is not a file
log = 'stderr'
q2cli.util.exit_with_error(e, header=header, traceback=log)
else:
cleanup_logfile = True
finally:
# OS X will reap temporary files that haven't been touched in
# 36 hours, double check that the log is still on the filesystem
# before trying to delete. Otherwise this will fail and the
# output won't be written.
if log and cleanup_logfile and os.path.exists(log.name):
log.close()
os.remove(log.name)
for result, output in zip(results, outputs):
path = result.save(output)
if not quiet:
click.secho('Saved %s to: %s' % (result.type, path),
fg='green')
def get_public_ruleset(save, outfile, ruleset_id):
"""Get a public ruleset by its RULESET_ID"""
ctx = click.get_current_context()
api = ctx.meta.get('api')
wdir = ctx.meta.get('wdir')
quiet = ctx.meta.get('quiet')
logger.info('Attempting to fetch ruleset %s', ruleset_id)
result = api.get_public_ruleset(ruleset_id=ruleset_id)
if not quiet:
click.echo(pygmentize_json(result))
if save:
if not outfile:
filepath = os.path.join(wdir, 'ruleset-{}.json'.format(ruleset_id))
outfile = io.open(filepath, 'wb')
else:
filepath = outfile.name
logger.info('Saving ruleset metadata to {}'.format(filepath))
json.dump(result, outfile)
logger.info('Ruleset metadata saved correctly')
def get_value(self, arguments, fallback=None):
import os
import os.path
import click
try:
path = self._locate_value(arguments, fallback=fallback)
# TODO: do we want a --force like flag?
if os.path.exists(path):
click.secho("Error: --%s directory already exists, won't "
"overwrite." % self.cli_name, err=True, fg='red',
bold=True)
ctx = click.get_current_context()
ctx.exit(1)
os.makedirs(path)
def fallback_(name, cli_name):
return os.path.join(path, name)
return fallback_
except ValueNotFoundException:
# Always fail to find a value as this handler doesn't exist.
def fail(*_):
raise ValueNotFoundException()
return fail
def process(ctx, yaml, no_yaml, templates, out, open_ren, open_text, args,
render, no_render, variable, force, quiet, include_tag):
"""
Preprocess text files, and render with pandoc or rmarkdown.
"""
def process_kv(variable):
kv_dict = {}
p = re.compile('^(?P<key>.+?):(?P<value>.+)$')
for kv in variable:
m = p.match(kv)
if m:
key = strip_quotes(m.group('key'))
value = strip_quotes(m.group('value'))
kv_dict[key] = value
else:
raise ValueError('Invalid key-value pair: {}.'.format(kv))
return kv_dict
def add_default_kv(d, identifier='compiled', key='id'):
return merge_dicts({key: identifier}, d)
def process_yaml(f):
identifier = FilePath.get_filename(f)
with open(f) as yamlfile:
yaml_vars = poyo.parse_string(yamlfile.read())
id_vars = add_default_kv(yaml_vars, identifier)
return id_vars
def generate(var_dict):
ren_path = chevron.render(out_path, var_dict)
ren_path_obj = FilePath(ren_path)
text_path_obj = FilePath(ren_path_obj.dir_ + ren_path_obj.name +
in_ext)
try:
gen_text(var_dict=var_dict,
ren_path_obj=ren_path_obj,
text_path_obj=text_path_obj)
if not no_render:
gen_ren(render,
ren_path_obj=ren_path_obj,
text_path_obj=text_path_obj)
else:
if text_path_obj.path != ren_path_obj.path:
shutil.copy(text_path_obj.path, ren_path_obj.path)
echo_success(ren_path_obj.basename + ' generated.')
except (RuntimeError, ValueError) as e:
raise click.ClickException(e.args)
click.Abort()
except (FileNotFoundError) as e:
raise click.ClickException(e.strerror + ': ' + e.filename)
click.Abort()
if open_ren:
open_file(ren_path_obj.path)
if open_text:
open_file(text_path_obj.path)
return
params = click.get_current_context().params
params.update(args=process_args(params['args']))
# if out is a dir
if re.fullmatch('.*/', out):
confirm(
'You supplied a directory for -o --out. No file name or format is specified. PDF files with default names will be created. \n\nOutput schema: {0}\n\nIs this correct?'
.format(out))
out_path = out + '{{id}}.pdf'
else:
out_path = out
in_ext = validate_exts(templates)
out_ext = os.path.splitext(out_path)[-1]
if out_ext == '':
confirm(
'\nThe output schema you supplied does not appear to have an extension. \n\nOutput schema: {0}\n\nIs this correct?'
.format(out_path))
if no_render:
if in_ext != out_ext:
confirm(
'\nYou choose to not render the documents, but the input and output files specified seem to have different extensions. \n\n Input: {input}\n Output: {output}\n\nIs this correct? If not, please modify -o --out.'
.format(input=' '.join(
[os.path.basename(t) for t in templates]),
output=os.path.basename(out_path)))
kv_dict = process_kv(variable)
all_yamls = []
if not no_yaml:
yaml_dicts = []
for y in yaml:
y_expanded = glob(y)
all_yamls += y_expanded
if not y_expanded:
if not variable:
echo_warn('Nothing matching {0}.'.format(y))
else:
for f in y_expanded:
yaml_dicts.append(process_yaml(f))
# assume that user forgot to add --no-yaml tag
if not yaml_dicts:
yaml_dicts = [add_default_kv({})]
# add kv pairs
gen_dicts = [merge_dicts(yd, kv_dict) for yd in yaml_dicts]
else:
if not kv_dict:
m = re.match('\{\{(?!id).+\}\}', out_path)
# contains anything other than {{id}}
if m:
raise click.BadOptionUsage(
option_name='-k --variable',
message=
'--no-yaml specified, but no key-value pairs provided.')
click.Abort()
gen_dicts = [add_default_kv(kv_dict)]
# multiple yamls but no variable in outpath
if len(gen_dicts) > 1 and not re.match('.*\{\{.+\}\}', out_path):
confirm(
'You have supplied multiple YAML files, but the output schema does not contain variables. Files generated earlier will be overriden.\n\nYAML files: {all_yamls} \nOutput schema: {out_path} \n\nIs this correct?'
.format(all_yamls=' '.join(all_yamls), out_path=out_path))
for kv in gen_dicts:
generate(kv)
def recognizer(model, pad, no_segmentation, bidi_reordering, script_ignore,
base_image, input, output, lines) -> None:
import json
import tempfile
from kraken import rpred
try:
im = Image.open(base_image)
except IOError as e:
raise click.BadParameter(str(e))
ctx = click.get_current_context()
# input may either be output from the segmenter then it is a JSON file or
# be an image file when running the OCR subcommand alone. might still come
# from some other subcommand though.
scripts = set()
if not lines and base_image != input:
lines = input
if not lines:
if no_segmentation:
lines = tempfile.NamedTemporaryFile(mode='w', delete=False)
logger.info(
'Running in no_segmentation mode. Creating temporary segmentation {}.'
.format(lines.name))
json.dump(
{
'script_detection': False,
'text_direction': 'horizontal-lr',
'boxes': [(0, 0) + im.size]
}, lines)
lines.close()
lines = lines.name
else:
raise click.UsageError(
'No line segmentation given. Add one with `-l` or run `segment` first.'
)
elif no_segmentation:
logger.warning(
'no_segmentation mode enabled but segmentation defined. Ignoring --no-segmentation option.'
)
with open_file(lines, 'r') as fp:
try:
fp = cast(IO[Any], fp)
bounds = json.load(fp)
except ValueError as e:
raise click.UsageError('{} invalid segmentation: {}'.format(
lines, str(e)))
# script detection
if bounds['script_detection']:
for l in bounds['boxes']:
for t in l:
scripts.add(t[0])
it = rpred.mm_rpred(model,
im,
bounds,
pad,
bidi_reordering=bidi_reordering,
script_ignore=script_ignore)
else:
it = rpred.rpred(model['default'],
im,
bounds,
pad,
bidi_reordering=bidi_reordering)
if not lines and no_segmentation:
logger.debug('Removing temporary segmentation file.')
os.unlink(lines.name)
preds = []
with log.progressbar(it, label='Processing',
length=len(bounds['boxes'])) as bar:
for pred in bar:
preds.append(pred)
ctx = click.get_current_context()
with open_file(output, 'w', encoding='utf-8') as fp:
fp = cast(IO[Any], fp)
message('Writing recognition results for {}\t'.format(base_image),
nl=False)
logger.info('Serializing as {} into {}'.format(ctx.meta['mode'],
output))
if ctx.meta['mode'] != 'text':
from kraken import serialization
fp.write(
serialization.serialize(preds, base_image,
Image.open(base_image).size,
ctx.meta['text_direction'], scripts,
ctx.meta['mode']))
else:
fp.write('\n'.join(s.prediction for s in preds))
message('\u2713', fg='green')
def shell():
exit_code = 0
with friendly_dockerized_errors(click):
shell_command = ShellCommand()
exit_code = shell_command.run()
click.get_current_context().exit(exit_code)
def get_user_context_obj():
"""Return user context, containing logging and configuration data.
:return: User context object (dict)
"""
return click.get_current_context().obj
def get_logger():
'get logger for current command.'
ctx = click.get_current_context()
qualname = _get_command_qualname(ctx)
logger = logging.getLogger(qualname)
return logger
def clean():
exit_code = 0
with friendly_dockerized_errors(click):
clean_command = CleanCommand()
exit_code = clean_command.run()
click.get_current_context().exit(exit_code)
def backup(clean, check):
"""Backup directory to HPSS"""
ctx = click.get_current_context()
run = ctx.parent.parent.params["run"]
ctx.parent.params[
"nmax"] = sys.maxsize # disable maximum launchers for backup
logger.warning("--nmax ignored for HPSS backup!")
directory = ctx.parent.params["directory"]
if not check and clean:
logger.error("Not running --clean without --check enabled.")
return ReturnCodes.ERROR
check_pattern()
logger.info("Discover launch directories ...")
block_launchers = load_block_launchers()
counter, nremove_total = 0, 0
os.chdir(directory)
for block, launchers in block_launchers.items():
logger.info(f"{block} with {len(launchers)} launcher(s)")
try:
isfile(f"{GARDEN}/{block}.tar")
except HpssOSError: # block not in HPSS
if run:
filelist = [os.path.join(block, l) for l in launchers]
args = shlex.split(
f"htar -M 5000000 -Phcvf {GARDEN}/{block}.tar")
try:
for line in run_command(args, filelist):
logger.info(line.strip())
except subprocess.CalledProcessError as e:
logger.error(str(e))
return ReturnCodes.ERROR
counter += 1
else:
logger.warning(f"Skip {block} - already in HPSS")
# Check backup here to allow running it separately
if check:
logger.info(f"Verify {block}.tar ...")
args = shlex.split(
f"htar -Kv -Hrelpaths -Hverify=all -f {GARDEN}/{block}.tar")
files_remove = []
try:
for line in run_command(args, []):
line = line.strip()
if line.startswith("HTAR: V "):
ls = line.split(", ")
if len(ls) == 3:
nfiles = len(files_remove)
if nfiles and not nfiles % 1000:
logger.info(f"{nfiles} files ...")
files_remove.append(ls[0].split()[-1])
else:
logger.info(line)
except subprocess.CalledProcessError as e:
logger.error(str(e))
return ReturnCodes.ERROR
if clean:
nremove = len(files_remove)
nremove_total += nremove
if run:
with click.progressbar(files_remove,
label="Removing files") as bar:
for fn in bar:
os.remove(fn)
logger.info(
f"Removed {nremove} files from disk for {block}.")
else:
logger.info(
f"Would remove {nremove} files from disk for {block}.")
logger.info(
f"{counter}/{len(block_launchers)} blocks newly backed up to HPSS.")
if clean:
if run:
logger.info(
f"Verified and removed a total of {nremove_total} files.")
else:
logger.info(
f"Would verify and remove a total of {nremove_total} files.")
return ReturnCodes.SUCCESS
def exec(command):
exit_code = 0
with friendly_dockerized_errors(click):
exec_command = ExecCommand(' '.join(command))
exit_code = exec_command.run()
click.get_current_context().exit(exit_code)
def cli(input, verbose):
ctx = click.get_current_context()
log.set_logger(logger, level=30-10*verbose)
ctx.meta['verbose'] = verbose
def compose(command):
exit_code = 0
with friendly_dockerized_errors(click):
compose_command = ComposeCommand(command)
exit_code = compose_command.run()
click.get_current_context().exit(exit_code)
def main(
template,
extra_context,
no_input,
checkout,
verbose,
replay,
overwrite_if_exists,
output_dir,
config_file,
default_config,
debug_file,
directory,
skip_if_file_exists,
accept_hooks,
replay_file,
list_installed,
):
"""Create a project from a Cookiecutter project template (TEMPLATE).
Cookiecutter is free and open source software, developed and managed by
volunteers. If you would like to help out or fund the project, please get
in touch at https://github.com/cookiecutter/cookiecutter.
"""
# Commands that should work without arguments
if list_installed:
list_installed_templates(default_config, config_file)
sys.exit(0)
# Raising usage, after all commands that should work without args.
if not template or template.lower() == 'help':
click.echo(click.get_current_context().get_help())
sys.exit(0)
configure_logger(stream_level='DEBUG' if verbose else 'INFO',
debug_file=debug_file)
# If needed, prompt the user to ask whether or not they want to execute
# the pre/post hooks.
if accept_hooks == "ask":
_accept_hooks = click.confirm("Do you want to execute hooks?")
else:
_accept_hooks = accept_hooks == "yes"
if replay_file:
replay = replay_file
try:
cookiecutter(
template,
checkout,
no_input,
extra_context=extra_context,
replay=replay,
overwrite_if_exists=overwrite_if_exists,
output_dir=output_dir,
config_file=config_file,
default_config=default_config,
password=os.environ.get('COOKIECUTTER_REPO_PASSWORD'),
directory=directory,
skip_if_file_exists=skip_if_file_exists,
accept_hooks=_accept_hooks,
)
except (
ContextDecodingException,
OutputDirExistsException,
InvalidModeException,
FailedHookException,
UnknownExtension,
InvalidZipRepository,
RepositoryNotFound,
RepositoryCloneFailed,
) as e:
click.echo(e)
sys.exit(1)
except UndefinedVariableInTemplate as undefined_err:
click.echo('{}'.format(undefined_err.message))
click.echo('Error message: {}'.format(undefined_err.error.message))
context_str = json.dumps(undefined_err.context,
indent=4,
sort_keys=True)
click.echo('Context: {}'.format(context_str))
sys.exit(1)
def main(nbpatterns, nbprescycles, homogenous, prestime, prestimetest, interpresdelay, patternsize, nbiter,
probadegrade, lr, print_every, rngseed):
#os.environ["CUDA_VISIBLE_DEVICES"] = "3"
torch.cuda.set_device(3)
train(paramdict=dict(click.get_current_context().params))
def get_datadir():
"""Fetch the current data directory."""
ctx = click.get_current_context()
return ctx.meta['renku.datasets.datadir']
def new_func(*args, **kwargs):
config_ = click.get_current_context().obj['config_file']
return f(config_, *args, **kwargs)
def run(inputdir):
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
datefmt='%d/%m/%Y %H:%M:%S',
stream=sys.stdout)
logging.getLogger().setLevel(logging.INFO)
context = click.get_current_context()
if not inputdir:
click.echo(run.get_help(context))
context.exit()
if not os.path.isdir(inputdir):
raise FileNotFoundError(inputdir + ' does not exists')
outputdir = os.path.realpath('../resources/images/cards')
logging.info('Checking output sub-dirs')
for card_variant in CARDS_VARIANTS:
card_variant_dir = os.path.join(outputdir, card_variant)
if not os.path.isdir(card_variant_dir):
logging.info('Creating output sub-dir ' + card_variant_dir)
os.mkdir(card_variant_dir)
card_paths = glob(os.path.join(inputdir, '*.png'))
logging.info('Processing cards')
for card_path in card_paths:
card_name = os.path.splitext(os.path.basename(card_path))[0]
if card_name.startswith('cardBack') or card_name == 'cardJoker':
logging.info('Ignoring ' + card_name)
continue
logging.info('Processing ' + card_name)
card_name = snake_case(card_name.replace('card', ''))
card_variant, card_type = card_name.split('_')
if card_variant not in CARDS_VARIANTS:
logging.info(card_name + ': invalid variant "' + card_variant +
'"')
continue
if card_type == 'a':
card_type = 'ace'
elif card_type == 'j':
card_type = 'jack'
elif card_type == 'k':
card_type = 'king'
elif card_type == 'q':
card_type = 'queen'
card_image = Image.open(card_path)
card_image.thumbnail((CARDS_WIDTH, 9999), Image.ANTIALIAS)
card_image.save(os.path.join(outputdir, card_variant,
card_type + '.png'),
optimize=True)
def execute_tasks(self, tasks, **kwargs):
ctx = click.get_current_context()
global_args = PropertyDict(kwargs)
# Show warning if some tasks are present several times in the list
dupplicated_tasks = sorted([
item for item, count in Counter(
task.name for task in tasks).items() if count > 1
])
if dupplicated_tasks:
dupes = ", ".join('"%s"' % t for t in dupplicated_tasks)
print(
"WARNING: Command%s found in the list of commands more than once. "
% ("s %s are" % dupes
if len(dupplicated_tasks) > 1 else " %s is" % dupes) +
"Only first occurence will be executed.")
# Set propagated global options.
# These options may be set on one subcommand, but available in the list of global arguments
for task in tasks:
for key in list(task.action_args):
option = next(
(o for o in ctx.command.params if o.name == key), None)
if option and (option.scope.is_global
or option.scope.is_shared):
local_value = task.action_args.pop(key)
global_value = global_args[key]
default = () if option.multiple else option.default
if global_value != default and local_value != default and global_value != local_value:
raise FatalError(
'Option "%s" provided for "%s" is already defined to a different value. '
"This option can appear at most once in the command line."
% (key, task.name))
if local_value != default:
global_args[key] = local_value
# Show warnings about global arguments
print_deprecation_warning(ctx)
# Make sure that define_cache_entry is mutable list and can be modified in callbacks
global_args.define_cache_entry = list(
global_args.define_cache_entry)
# Execute all global action callback - first from idf.py itself, then from extensions
for action_callback in ctx.command.global_action_callbacks:
action_callback(ctx, global_args, tasks)
# Always show help when command is not provided
if not tasks:
print(ctx.get_help())
ctx.exit()
# Build full list of tasks to and deal with dependencies and order dependencies
tasks_to_run = OrderedDict()
while tasks:
task = tasks[0]
tasks_dict = dict([(t.name, t) for t in tasks])
dependecies_processed = True
# If task have some dependecies they have to be executed before the task.
for dep in task.dependencies:
if dep not in tasks_to_run.keys():
# If dependent task is in the list of unprocessed tasks move to the front of the list
if dep in tasks_dict.keys():
dep_task = tasks.pop(tasks.index(tasks_dict[dep]))
# Otherwise invoke it with default set of options
# and put to the front of the list of unprocessed tasks
else:
print(
'Adding "%s"\'s dependency "%s" to list of commands with default set of options.'
% (task.name, dep))
dep_task = ctx.invoke(
ctx.command.get_command(ctx, dep))
# Remove options with global scope from invoke tasks because they are alread in global_args
for key in list(dep_task.action_args):
option = next((o for o in ctx.command.params
if o.name == key), None)
if option and (option.scope.is_global
or option.scope.is_shared):
dep_task.action_args.pop(key)
tasks.insert(0, dep_task)
dependecies_processed = False
# Order only dependencies are moved to the front of the queue if they present in command list
for dep in task.order_dependencies:
if dep in tasks_dict.keys(
) and dep not in tasks_to_run.keys():
tasks.insert(0,
tasks.pop(tasks.index(tasks_dict[dep])))
dependecies_processed = False
if dependecies_processed:
# Remove task from list of unprocessed tasks
tasks.pop(0)
# And add to the queue
if task.name not in tasks_to_run.keys():
tasks_to_run.update([(task.name, task)])
# Run all tasks in the queue
# when global_args.dry_run is true idf.py works in idle mode and skips actual task execution
if not global_args.dry_run:
for task in tasks_to_run.values():
name_with_aliases = task.name
if task.aliases:
name_with_aliases += " (aliases: %s)" % ", ".join(
task.aliases)
print("Executing action: %s" % name_with_aliases)
task.run(ctx, global_args, task.action_args)
self._print_closing_message(global_args, tasks_to_run.keys())
return tasks_to_run
def inner(*args, **kwargs):
ctx = click.get_current_context()
for arg in ctx.help_option_names:
if arg in sys.argv:
return
f(*args, **kwargs)
def main(nbclasses, alpha, rule, gamma, steplr, activ, flare, nbshots, nbf,
prestime, prestimetest, ipd, nbiter, lr, test_every, save_every,
rngseed):
train(paramdict=dict(click.get_current_context().params))
def parse(task_ids, nproc, store_volumetric_data):
"""Parse VASP launchers into tasks"""
ctx = click.get_current_context()
if "CLIENT" not in ctx.obj:
raise EmmetCliError("Use --spec to set target DB for tasks!")
run = ctx.parent.parent.params["run"]
nmax = ctx.parent.params["nmax"]
directory = ctx.parent.params["directory"].rstrip(os.sep)
tag = os.path.basename(directory)
target = ctx.obj["CLIENT"]
logger.info(
f"Connected to {target.collection.full_name} with {target.collection.count()} tasks."
)
ensure_indexes(["task_id", "tags", "dir_name", "retired_task_id"],
[target.collection])
chunk_size = math.ceil(nmax / nproc)
if nproc > 1 and nmax <= chunk_size:
nproc = 1
logger.warning(
f"nmax = {nmax} but chunk size = {chunk_size} -> sequential parsing."
)
pool = multiprocessing.Pool(processes=nproc)
gen = VaspDirsGenerator()
iterator = iterator_slice(gen, chunk_size) # process in chunks
queue = deque()
count = 0
sep_tid = None
if task_ids:
with open(task_ids, "r") as f:
task_ids = json.load(f)
else:
# reserve list of task_ids to avoid collisions during multiprocessing
# insert empty doc with max ID + 1 into target collection for parallel SLURM jobs
# NOTE use regex first to reduce size of distinct below 16MB
all_task_ids = target.collection.distinct(
"task_id", {"task_id": {
"$regex": r"^mp-\d{7,}$"
}})
if not all_task_ids:
all_task_ids = target.collection.distinct("task_id")
next_tid = max(int(tid.split("-")[-1]) for tid in all_task_ids) + 1
lst = [f"mp-{next_tid + n}" for n in range(nmax)]
if run:
sep_tid = f"mp-{next_tid + nmax}"
target.collection.insert({"task_id": sep_tid})
logger.info(f"Inserted separator task with task_id {sep_tid}.")
task_ids = chunks(lst, chunk_size)
logger.info(f"Reserved {len(lst)} task ID(s).")
while iterator or queue:
try:
args = [next(iterator), tag, task_ids]
queue.append(pool.apply_async(parse_vasp_dirs, args))
except (StopIteration, TypeError):
iterator = None
while queue and (len(queue) >= pool._processes or not iterator):
process = queue.pop()
process.wait(1)
if not process.ready():
queue.append(process)
else:
count += process.get()
pool.close()
if run:
logger.info(
f"Successfully parsed and inserted {count}/{gen.value} tasks in {directory}."
)
if sep_tid:
target.collection.remove({"task_id": sep_tid})
logger.info(f"Removed separator task {sep_tid}.")
else:
logger.info(
f"Would parse and insert {count}/{gen.value} tasks in {directory}."
)
return ReturnCodes.SUCCESS if count and gen.value else ReturnCodes.WARNING
def push():
exit_code = 0
with friendly_dockerized_errors(click):
push_command = PushCommand()
exit_code = push_command.run()
click.get_current_context().exit(exit_code)
def wrapper(*args, **kwargs):
ctx = click.get_current_context()
cli_config = CliConfig(**kwargs)
return ctx.invoke(func, cli_config, *args[1:], **kwargs)
def main():
ctx = click.get_current_context()
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
sys.exit(0)
def build_environment(
generate_packages: bool,
build_environment: bool,
environment_name: str,
environment_file: str,
package_version_file: str,
start_from_wave: int,
stop_at_wave: int,
) -> None:
if not generate_packages and not build_environment:
ctx = click.get_current_context()
click.echo(ctx.get_help())
ctx.exit()
if generate_packages:
rich.print("[bold]Regenerating packages[/bold]")
rich.print("[bold]-----[/bold]")
if not package_version_file:
raise RuntimeError(
"Please specify the version file to use with the "
"--package_version_file option")
with open(pathlib.Path(package_version_file)) as open_file:
version_dict: Dict[str, str] = yaml.load(open_file,
Loader=yaml.SafeLoader)
wave_index: int
wave: Dict[str, str]
for wave_index, wave in enumerate(repository_data):
if wave_index < start_from_wave:
continue
if wave_index > stop_at_wave:
continue
rich.print(f"[bold]Building wave {wave_index}...[/bold]")
rich.print(
f"[bold]Packages in the wave: {tuple(wave.keys())}[/bold]")
start_time: datetime.datetime = datetime.datetime.now()
packages_to_build: Dict[str, feedstock_utils.Repository] = {}
package_build_status: Dict[str, BuildStatus] = {}
package: str
for package in wave:
if package not in version_dict:
raise RuntimeError(
f"Package {package} not found in version file")
for package in wave:
repository: feedstock_utils.Repository = feedstock_utils.Repository(
repository=wave[package], debug=False)
repository.set_version_and_build_number(package, version_dict)
repository.run_conda_smithy()
repository.git_push()
packages_to_build[package] = repository
package_build_status[package] = BuildStatus(
status="Not Started Yet", last_run_id=0)
wave_successful = build_wave(packages_to_build,
package_build_status, start_time)
if not wave_successful:
rich.print(
f"[bold]Wave {wave_index} did not build correctly: stopping[/bold]"
)
if build_environment:
rich.print("[bold]\nBuilding environment[/bold]")
if not environment_name:
raise RuntimeError(
"Please specify the environment name with the --environment-name "
"option")
if not environment_file:
raise RuntimeError(
"Please specify the file describing the environment with the "
"--environment-file option")
rich.print(
f"[bold]Building environment {environment_name} based on file "
f"{environment_file}[/bold]")
rich.print(
f"conda env create -n {environment_name} --file {environment_file}"
)
return_code = subprocess.call(
f"conda env create -n {environment_name} --file {environment_file}",
shell=True,
)
if return_code != 0:
raise RuntimeError(f"Error building {environment_name} repository")
def set_global_options(json_, force=False, wait=False):
ctx = click.get_current_context()
ctx.obj[CTX_FORCE_COMPATIBILITY] = force
ctx.obj[CTX_BLOCKING_MODE] = wait
ctx.obj[CTX_OUTPUT_JSON] = json_
def exit_with_msg():
ctx = click.get_current_context()
ctx.fail("target and email is required!")
def custom_except_hook(exc_info):
"""
A custom excepthook to present python errors produced by the CLI.
We don't want to show end users big scary stacktraces if they aren't python
programmers, so slim it down to some basic info. We keep a "DEBUGMODE" env
variable kicking around to let us turn on stacktraces if we ever need them.
Additionally, does global suppression of EPIPE errors, which often occur
when a python command is piped to a consumer like `head` which closes its
input stream before python has sent all of its output.
DANGER: There is a (small) risk that this will bite us if there are EPIPE
errors produced within the Globus SDK. We should keep an eye on this
possibility, as it may demand more sophisticated handling of EPIPE.
Possible TODO item to reduce this risk: inspect the exception and only hide
EPIPE if it comes from within the globus_cli package.
"""
exception_type, exception, traceback = exc_info
# check if we're in debug mode, and run the real excepthook if we are
ctx = click.get_current_context()
state = ctx.ensure_object(CommandState)
if state.debug:
sys.excepthook(exception_type, exception, traceback)
# we're not in debug mode, do custom handling
else:
# if it's a click exception, re-raise as original -- Click's main
# execution context will handle pretty-printing
if isinstance(exception, click.ClickException):
reraise(exception_type, exception, traceback)
# handle the Globus-raised errors with our special hooks
# these will present the output (on stderr) as JSON
elif isinstance(exception, exc.TransferAPIError):
if exception.code == "ClientError.AuthenticationFailed":
authentication_hook(exception)
else:
transferapi_hook(exception)
elif isinstance(exception, exc.AuthAPIError):
if exception.code == "UNAUTHORIZED":
authentication_hook(exception)
# invalid_grant occurs when the users refresh tokens are not valid
elif exception.message == "invalid_grant":
invalidrefresh_hook(exception)
else:
authapi_hook(exception)
elif isinstance(exception, exc.GlobusAPIError):
globusapi_hook(exception)
# specific checks fell through -- now check if it's any kind of
# GlobusError
elif isinstance(exception, exc.GlobusError):
globus_generic_hook(exception)
# not a GlobusError, not a ClickException -- something like ValueError
# or NotImplementedError bubbled all the way up here: just print it
# out, basically
else:
safeprint(u'{}: {}'.format(exception_type.__name__, exception))
sys.exit(1)
def cli(broker_url, port): # pylint: disable=unused-argument
ctx = click.get_current_context()
Exporter().run(ctx.params)
def repl():
"""Start an interactive session"""
prompt_kwargs = {
'history': FileHistory(os.path.expanduser('~/.repl_history'))
}
click_repl.repl(click.get_current_context(), prompt_kwargs=prompt_kwargs)
def friendly_dockerized_errors(click):
try:
yield
except CommandError as err:
click.echo(err.message, err=True)
click.get_current_context().exit(1)
def restore(inputfile, file_filter):
"""Restore launchers from HPSS"""
ctx = click.get_current_context()
run = ctx.parent.parent.params["run"]
nmax = ctx.parent.params["nmax"]
pattern = ctx.parent.params["pattern"]
directory = ctx.parent.params["directory"]
if not os.path.exists(directory):
os.makedirs(directory)
check_pattern(nested_allowed=True)
shutil.chown(directory, group="matgen")
block_launchers = defaultdict(list)
nlaunchers = 0
with open(inputfile, "r") as infile:
os.chdir(directory)
with click.progressbar(infile, label="Load blocks") as bar:
for line in bar:
if fnmatch(line, pattern):
if nlaunchers == nmax:
break
block, launcher = line.split(os.sep, 1)
for ff in file_filter:
block_launchers[block].append(
os.path.join(launcher.strip(), ff))
nlaunchers += 1
nblocks = len(block_launchers)
nfiles = sum(len(v) for v in block_launchers.values())
logger.info(f"Restore {nblocks} block(s) with {nlaunchers} launchers"
f" and {nfiles} file filters to {directory} ...")
nfiles_restore_total, max_args = 0, 15000
for block, files in block_launchers.items():
# get full list of matching files in archive and check against existing files
args = shlex.split(f"htar -tf {GARDEN}/{block}.tar")
filelist = [os.path.join(block, f) for f in files]
filelist_chunks = [
filelist[i:i + max_args] for i in range(0, len(filelist), max_args)
]
filelist_restore, cnt = [], 0
try:
for chunk in filelist_chunks:
for line in run_command(args, chunk):
fn = extract_filename(line)
if fn:
cnt += 1
if os.path.exists(fn):
logger.debug(
f"Skip {fn} - already exists on disk.")
else:
filelist_restore.append(fn)
except subprocess.CalledProcessError as e:
logger.error(str(e))
return ReturnCodes.ERROR
# restore what's missing
if filelist_restore:
nfiles_restore = len(filelist_restore)
nfiles_restore_total += nfiles_restore
if run:
logger.info(
f"Restore {nfiles_restore}/{cnt} files for {block} to {directory} ..."
)
args = shlex.split(f"htar -xvf {GARDEN}/{block}.tar")
filelist_restore_chunks = [
filelist_restore[i:i + max_args]
for i in range(0, len(filelist_restore), max_args)
]
try:
for chunk in filelist_restore_chunks:
for line in run_command(args, chunk):
logger.info(line.strip())
except subprocess.CalledProcessError as e:
logger.error(str(e))
return ReturnCodes.ERROR
else:
logger.info(
f"Would restore {nfiles_restore}/{cnt} files for {block} to {directory}."
)
else:
logger.warning(f"Nothing to restore for {block}!")
if run:
logger.info(f"Set group of {block} to matgen recursively ...")
recursive_chown(block, "matgen")
if run:
logger.info(f"Restored {nfiles_restore_total} files to {directory}.")
else:
logger.info(
f"Would restore {nfiles_restore_total} files to {directory}.")
return ReturnCodes.SUCCESS
