def eval_var(var): """Wrap ast.literal_eval to provide more helpful error. Examples: >>> eval_var('42') 42 >>> eval_var('"string"') 'string' >>> eval_var('string') Traceback (most recent call last): cylc.flow.exceptions.UserInputError: Invalid template variable: string (note string values must be quoted) >>> eval_var('[') Traceback (most recent call last): cylc.flow.exceptions.UserInputError: Invalid template variable: [ (values must be valid Python literals) """ try: return literal_eval(var) except ValueError: raise UserInputError(f'Invalid template variable: {var}' '\n(note string values must be quoted)') from None except SyntaxError: raise UserInputError( f'Invalid template variable: {var}' '\n(values must be valid Python literals)') from None
def parse_rm_dirs(rm_dirs: Iterable[str]) -> Set[str]: """Parse a list of possibly colon-separated dirs (or files or globs). Return the set of all the dirs. Used by cylc clean with the --rm option. """ result: Set[str] = set() for item in rm_dirs: for part in item.split(':'): part = part.strip() if not part: continue is_dir = part.endswith(os.sep) part = os.path.normpath(part) if os.path.isabs(part): raise UserInputError("--rm option cannot take absolute paths") if part == '.' or part.startswith(f'..{os.sep}'): raise UserInputError( "--rm option cannot take paths that point to the " "run directory or above") if is_dir: # Preserve trailing slash to ensure it only matches dirs, # not files, when globbing part += os.sep result.add(part) return result
def _validate(options: 'Values', *task_globs: str) -> None: """Check combination of options and task globs is valid.""" if options.release_all: if task_globs: raise UserInputError("Cannot combine --all with TASK_GLOB(s).") else: if not task_globs: raise UserInputError("Missing arguments: TASK_GLOB [...]. " "See `cylc release --help`.")
def _validate(options: 'Values', *tokens_list: str) -> None: """Check combination of options and task globs is valid.""" if options.release_all: if tokens_list: raise UserInputError("Cannot combine --all with Cycle/Task IDs") else: if not tokens_list: raise UserInputError( "Must define Cycles/Tasks. See `cylc release --help`.")
def _validate(options: 'Values', *task_globs: str) -> None: """Check combination of options and task globs is valid.""" if options.hold_point_string: if task_globs: raise UserInputError( "Cannot combine --after with TASK_GLOB(s).\n" "`cylc hold --after` holds all tasks after the given " "cycle point.") elif not task_globs: raise UserInputError( "Missing arguments: TASK_GLOB [...]. See `cylc hold --help`.")
def _validate(options: 'Values', *task_globs: str) -> None: """Check combination of options and task globs is valid.""" if options.hold_point_string: if task_globs: raise UserInputError( "Cannot combine --after with Cylc/Task IDs.\n" "`cylc hold --after` holds all tasks after the given " "cycle point.") elif not task_globs: raise UserInputError( "Must define Cycles/Tasks. See `cylc hold --help`.")
def install(parser: COP, opts: 'Values', reg: Optional[str] = None) -> None: if opts.no_run_name and opts.run_name: raise UserInputError( "options --no-run-name and --run-name are mutually exclusive.") if reg is None: source = opts.source else: if opts.source: raise UserInputError( "WORKFLOW_NAME and --directory are mutually exclusive.") source = search_install_source_dirs(reg) workflow_name = opts.workflow_name or reg for entry_point in iter_entry_points('cylc.pre_configure'): try: if source: entry_point.resolve()(srcdir=source, opts=opts) else: from pathlib import Path entry_point.resolve()(srcdir=Path().cwd(), opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors raise PluginError('cylc.pre_configure', entry_point.name, exc) from None cli_symdirs: Optional[Dict[str, Dict[str, Any]]] = None if opts.symlink_dirs == '': cli_symdirs = {} elif opts.symlink_dirs: cli_symdirs = parse_cli_sym_dirs(opts.symlink_dirs) source_dir, rundir, _workflow_name = install_workflow( workflow_name=workflow_name, source=source, run_name=opts.run_name, no_run_name=opts.no_run_name, cli_symlink_dirs=cli_symdirs) for entry_point in iter_entry_points('cylc.post_install'): try: entry_point.resolve()(srcdir=source_dir, opts=opts, rundir=str(rundir)) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors raise PluginError('cylc.post_install', entry_point.name, exc) from None
def main(parser, opts, suite=None, start=None, stop=None): """Implement ``cylc graph``.""" if opts.ungrouped and opts.namespaces: raise UserInputError('Cannot combine --ungrouped and --namespaces.') if not opts.reference: raise UserInputError('Only the --reference use cases are supported') template_vars = load_template_vars( opts.templatevars, opts.templatevars_file) config = get_config(suite, opts, template_vars=template_vars) if opts.namespaces: graph_inheritance(config) else: graph_workflow(config, start, stop, ungrouped=opts.ungrouped, show_suicide=opts.show_suicide)
def get_rdict(left, right=None): """Check+transform left=right into a nested dict. left can be key, [key], [key1]key2, [key1][key2], [key1][key2]key3, etc. """ if left == "inherit": raise UserInputError( "Inheritance cannot be changed by broadcast") rdict = {} cur_dict = rdict tail = left while tail: match = REC_ITEM.match(tail) if match: sect, tail = match.groups() if tail: # [sect]... = right cur_dict[sect.strip()] = {} cur_dict = cur_dict[sect.strip()] else: # [sect] = right cur_dict[sect.strip()] = right else: # item = right cur_dict[tail.strip()] = right tail = None upg({'runtime': {'__MANY__': rdict}}, 'test') cylc_config_validate(rdict, SPEC['runtime']['__MANY__']) return rdict
async def main(opts, color=False, scan_dir=None, write=cprint): """Open up a Python API for testing purposes. Note: Don't use this API for anything other than testing, there is a proper Python API for these purposes. """ # validate / standardise the list of workflow states opts.states = set(opts.states.split(',')) if 'all' in opts.states: opts.states = FLOW_STATES elif (not opts.states or not all(state in FLOW_STATES for state in opts.states)): raise UserInputError( '--states must be set to a comma separated list of workflow' ' states. \nSee `cylc scan --help` for a list of supported' ' states.') if not color: # we cannot support colour or have been requested not to use it opts.colour_blind = True # print state totals key as needed if opts.format == 'rich' and not opts.colour_blind: cprint(state_totals_key() + '\n') await scanner(opts, write, scan_dir)
def _validate_constraint(*tokens_list, constraint=None): if constraint == 'workflows': for tokens in tokens_list: if tokens.is_null or tokens.is_task_like: raise UserInputError('IDs must be workflows') return if constraint == 'tasks': for tokens in tokens_list: if tokens.is_null or not tokens.is_task_like: raise UserInputError('IDs must be tasks') return if constraint == 'mixed': for tokens in tokens_list: if tokens.is_null: raise UserInputError('IDs cannot be null.') return
def _validate_workflow_ids(*tokens_list, src_path): for ind, tokens in enumerate(tokens_list): if tokens['user']: raise UserInputError( "Operating on others users' workflows is not supported") if not src_path: validate_workflow_name(tokens['workflow']) if ind == 0 and src_path: # source workflow passed in as a path pass else: src_path = Path(get_workflow_run_dir(tokens['workflow'])) if src_path.is_file(): raise UserInputError( f'Workflow ID cannot be a file: {tokens["workflow"]}') detect_both_flow_and_suite(src_path)
def _validate_number(*tokens_list, max_workflows=None, max_tasks=None): if not max_workflows and not max_tasks: return workflows_count = 0 tasks_count = 0 for tokens in tokens_list: if tokens.is_task_like: tasks_count += 1 else: workflows_count += 1 if max_workflows and workflows_count > max_workflows: raise UserInputError( f'IDs contain too many workflows (max {max_workflows})') if max_tasks and tasks_count > max_tasks: raise UserInputError( f'IDs contain too many cycles/tasks/jobs (max {max_tasks})')
def main(parser: COP, options: 'Values', *ids: str): """CLI for "cylc trigger".""" if options.flow_descr and not options.reflow: raise UserInputError("--meta requires --reflow") call_multi( partial(run, options), *ids, )
def main(_, opts: 'Values', *ids: str): if cylc.flow.flags.verbosity < 2: disable_timestamps(LOG) if opts.local_only and opts.remote_only: raise UserInputError( "--local and --remote options are mutually exclusive") asyncio.run(run(*ids, opts=opts))
def main(parser: COP, options: 'Values', *args: str) -> None: """CLI.""" if not args: return parser.error('No message supplied') if len(args) <= 2: # BACK COMPAT: args <= 2 # from: # 7.6? # remove at: # 9.0? # (As of Dec 2020 some functional tests still use the classic # two arg interface) workflow_id = os.getenv('CYLC_WORKFLOW_ID') task_job = os.getenv('CYLC_TASK_JOB') message_strs = list(args) else: workflow_id, task_job, *message_strs = args workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) # Read messages from STDIN if '-' in message_strs: current_message_str = '' while True: # Note: `for line in sys.stdin:` can hang message_str = sys.stdin.readline() if message_str.strip(): # non-empty line current_message_str += message_str elif message_str: # empty line, start next message if current_message_str: message_strs.append(current_message_str) current_message_str = '' # reset else: # end of file if current_message_str: message_strs.append(current_message_str) break # Separate "severity: message" messages = [] # [(severity, message_str), ...] for message_str in message_strs: if message_str == '-': pass elif ':' in message_str: valid, err_msg = TaskMessageValidator.validate(message_str) if not valid: raise UserInputError( f'Invalid task message "{message_str}" - {err_msg}') messages.append( [item.strip() for item in message_str.split(':', 1)]) elif options.severity: messages.append([options.severity, message_str.strip()]) else: messages.append([getLevelName(INFO), message_str.strip()]) record_messages(workflow_id, task_job, messages)
def main(parser: COP, options: 'Values', severity_str: str, *ids) -> None: try: severity = LOG_LEVELS[severity_str] except KeyError: raise UserInputError("Illegal logging level, %s" % severity_str) call_multi( partial(run, options, severity), *ids, constraint='workflows', )
def main(parser: COP, opts: 'Values', reg: str): # Note: do not use workflow_files.parse_reg here if cylc.flow.flags.verbosity < 2: disable_timestamps(LOG) if opts.local_only and opts.remote_only: raise UserInputError( "--local and --remote options are mutually exclusive") init_clean(reg, opts)
def main(parser, options, *args): """CLI.""" if not args: return parser.error('No message supplied') cylc.flow.flags.verbose = os.getenv('CYLC_VERBOSE') == 'true' cylc.flow.flags.debug = os.getenv('CYLC_DEBUG') == 'true' if len(args) <= 2: # BACK COMPAT: args <= 2 # from: # 7.6? # remove at: # 9.0? # (As of Dec 2020 some functional tests still use the classic # two arg interface) suite = os.getenv('CYLC_SUITE_NAME') task_job = os.getenv('CYLC_TASK_JOB') message_strs = list(args) else: suite, task_job, *message_strs = args # Read messages from STDIN if '-' in message_strs: current_message_str = '' while True: # Note: `for line in sys.stdin:` can hang message_str = sys.stdin.readline() if message_str.strip(): # non-empty line current_message_str += message_str elif message_str: # empty line, start next message if current_message_str: message_strs.append(current_message_str) current_message_str = '' # reset else: # end of file if current_message_str: message_strs.append(current_message_str) break # Separate "severity: message" messages = [] # [(severity, message_str), ...] for message_str in message_strs: if message_str == '-': pass elif ':' in message_str: valid, err_msg = TaskMessageValidator.validate(message_str) if not valid: raise UserInputError( f'Invalid task message "{message_str}" - {err_msg}') messages.append( [item.strip() for item in message_str.split(':', 1)]) elif options.severity: messages.append([options.severity, message_str.strip()]) else: messages.append([getLevelName(INFO), message_str.strip()]) record_messages(suite, task_job, messages)
def parse_args(): if {'help', '--help', "-h"} & set(sys.argv): print(__doc__) elif len(sys.argv) < 2: raise UserInputError( "wrong number of arguments, " f"see '{os.path.basename(sys.argv[0])} --help'.") elif '--list' in sys.argv: print('\n'.join(list_resources())) else: return (None, sys.argv[1:]) sys.exit()
def main(parser, opts, workflow=None, start=None, stop=None): """Implement ``cylc graph``.""" if opts.ungrouped and opts.namespaces: raise UserInputError('Cannot combine --ungrouped and --namespaces.') if not (opts.reference or opts.diff): raise UserInputError( 'Only the --reference and --diff use cases are supported') template_vars = load_template_vars(opts.templatevars, opts.templatevars_file) write = print flows = [(workflow, [])] if opts.diff: flows.append((opts.diff, [])) for flow, graph in flows: if opts.diff: write = graph.append config = get_config(flow, opts, template_vars=template_vars) if opts.namespaces: graph_inheritance(config, write=write) else: graph_workflow(config, start, stop, ungrouped=opts.ungrouped, show_suicide=opts.show_suicide, write=write) if opts.diff: lines = list( unified_diff([f'{line}\n' for line in flows[0][1]], [f'{line}\n' for line in flows[1][1]], fromfile=flows[0][0], tofile=flows[1][0])) if lines: sys.stdout.writelines(lines) sys.exit(1)
def main(parser: COP, opts: 'Values', workflow: str, start: Optional[str] = None, stop: Optional[str] = None) -> None: """Implement ``cylc graph``.""" if opts.grouping and opts.namespaces: raise UserInputError('Cannot combine --group and --namespaces.') lines: List[str] = [] if not (opts.reference or opts.diff): write = lines.append else: write = print flows: List[Tuple[str, List[str]]] = [(workflow, [])] if opts.diff: flows.append((opts.diff, [])) for flow, graph in flows: if opts.diff: write = graph.append config = get_config(flow, opts) if opts.namespaces: graph_inheritance(config, write=write) else: graph_workflow(config, start, stop, grouping=opts.grouping, show_suicide=opts.show_suicide, write=write) if opts.diff: diff_lines = list( unified_diff([f'{line}\n' for line in flows[0][1]], [f'{line}\n' for line in flows[1][1]], fromfile=flows[0][0], tofile=flows[1][0])) if diff_lines: sys.stdout.writelines(diff_lines) sys.exit(1) if not (opts.reference or opts.diff): filename = dot(opts, lines) if opts.output: print(f'Graph rendered to {opts.output}') else: gui(filename)
def load(config, additional_plugins=None): additional_plugins = additional_plugins or [] entry_points = { entry_point.name: entry_point for entry_point in iter_entry_points('cylc.main_loop') } plugins = { 'state': {}, 'timings': {} } for plugin_name in config['plugins'] + additional_plugins: # get plugin try: module_name = entry_points[plugin_name.replace(' ', '_')] except KeyError: raise UserInputError( f'No main-loop plugin: "{plugin_name}"\n' + ' Available plugins:\n' + indent('\n'.join(sorted(entry_points)), ' ') ) # load plugin try: module = module_name.load() except Exception: raise CylcError(f'Could not load plugin: "{plugin_name}"') # load coroutines log = [] for coro_name, coro in ( (coro_name, coro) for coro_name, coro in getmembers(module) if isfunction(coro) if hasattr(coro, 'main_loop') ): log.append(coro_name) plugins.setdefault( coro.main_loop, {} )[(plugin_name, coro_name)] = coro plugins['timings'][(plugin_name, coro_name)] = deque(maxlen=1) LOG.debug( 'Loaded main loop plugin "%s": %s', plugin_name + '\n', '\n'.join((f'* {x}' for x in log)) ) # set the initial state of the plugin plugins['state'][plugin_name] = {} # make a note of the config here for ease of reference plugins['config'] = config return plugins
async def _expand_workflow_tokens_impl(tokens, match_active=True): """Use "cylc scan" to expand workflow patterns.""" workflow_sel = tokens['workflow_sel'] if workflow_sel and workflow_sel != 'running': raise UserInputError(f'The workflow selector :{workflow_sel} is not' 'currently supported.') # construct the pipe pipe = scan | filter_name(fnmatch.translate(tokens['workflow'])) if match_active is not None: pipe |= is_active(match_active) # iter the results async for workflow in pipe: yield tokens.duplicate(workflow=workflow['name'])
def main(parser: COP, opts: 'Values', reg: str): # Note: do not use workflow_files.parse_reg here if cylc.flow.flags.verbosity < 2: # for readability omit timestamps from logging unless in debug mode for handler in LOG.handlers: if isinstance(handler.formatter, CylcLogFormatter): handler.formatter.configure(timestamp=False) if opts.local_only and opts.remote_only: raise UserInputError( "--local and --remote options are mutually exclusive" ) init_clean(reg, opts)
def _parse_src_path(id_): src_path = Path(id_) if (id_ == os.curdir or id_.startswith(f'{os.curdir}{os.sep}') or Path(id_).is_absolute()): src_path = src_path.resolve() if not src_path.exists(): raise UserInputError(f'Path does not exist: {src_path}') if src_path.name in {WorkflowFiles.FLOW_FILE, WorkflowFiles.SUITE_RC}: src_path = src_path.parent try: src_file_path = check_flow_file(src_path) except WorkflowFilesError: raise WorkflowFilesError(NO_FLOW_FILE_MSG.format(id_)) workflow_id = src_path.name return workflow_id, src_path, src_file_path return None
def main(parser, options, suite, *items): for i, item in enumerate(items): if not TaskID.is_valid_id_2(item): raise UserInputError( '"%s": invalid task ID (argument %d)' % (item, i + 1)) prompt('Insert %s in %s' % (items, suite), options.force) pclient = SuiteRuntimeClient( suite, options.owner, options.host, options.port) pclient( 'insert_tasks', {'items': items, 'no_check': options.no_check, 'stop_point_string': options.stop_point_string}, timeout=options.comms_timeout )
def main( parser: COP, options: 'Values', workflow: str, task_id: Optional[str] = None ) -> None: workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) if task_id and not TaskID.is_valid_id(task_id): raise UserInputError("Invalid task ID: %s" % task_id) flow_kwargs = { 'request_string': FLOW_QUERY, 'variables': {'wFlows': [workflow]} } task_kwargs: Dict[str, Any] = { 'request_string': TASK_QUERY, } # cylc ping WORKFLOW result = pclient('graphql', flow_kwargs) msg = "" for flow in result['workflows']: w_name = flow['name'] w_port = flow['port'] w_pub_port = flow['pubPort'] if cylc.flow.flags.verbosity > 0: sys.stdout.write( f'{w_name} running on ' f'{pclient.host}:{w_port} {w_pub_port}\n' ) # cylc ping WORKFLOW TASKID if task_id: task, point = TaskID.split(task_id) w_id = flow['id'] task_kwargs['variables'] = { 'tProxy': f'{w_id}{ID_DELIM}{point}{ID_DELIM}{task}' } task_result = pclient('graphql', task_kwargs) if not task_result.get('taskProxy'): msg = "task not found" elif task_result['taskProxy']['state'] != TASK_STATUS_RUNNING: msg = f"task not {TASK_STATUS_RUNNING}" if msg: print(cparse(f'<red>{msg}</red>')) sys.exit(1)
def main( parser: COP, options: 'Values', *ids, ) -> None: if options.print_platform_names and options.print_platforms: options.print_platform_names = False if options.print_platform_names or options.print_platforms: # Get platform information: if ids: raise UserInputError( "Workflow IDs are incompatible with --platform options.") glbl_cfg().platform_dump(options.print_platform_names, options.print_platforms) return if not ids: if options.print_hierarchy: print("\n".join(get_config_file_hierarchy())) return glbl_cfg().idump(options.item, not options.defaults, oneline=options.oneline, none_str=options.none_str) return workflow_id, _, flow_file = parse_id( *ids, src=True, constraint='workflows', ) if options.print_hierarchy: print("\n".join(get_config_file_hierarchy(workflow_id))) return config = WorkflowConfig(workflow_id, flow_file, options, get_template_vars(options)) config.pcfg.idump(options.item, not options.defaults, oneline=options.oneline, none_str=options.none_str)
def get_resources(resource: str, tgt_dir: Optional[str]): """Extract cylc.flow resources and write them to a target directory. Arguments: resource: path relative to RESOURCE_DIR. target_dir: Where to put extracted resources, created if necessary. """ # get the resource path resource_path = Path(resource) if resource in ('api-key', 'tutorial/api-key'): print(get_api_key()) return src = RESOURCE_DIR / resource_path if not src.exists(): raise UserInputError( f'No such resources {resource}.' '\nRun `cylc get-resources --list` for resource names.') is_tutorial = path_is_tutorial(src) # get the target path if not tgt_dir: if is_tutorial: # this is a tutorial => use the primary source dir _tgt_dir = Path(glbl_cfg().get(['install', 'source dirs'])[0]) else: # this is a regular resource => use $PWD _tgt_dir = Path.cwd() else: _tgt_dir = Path(tgt_dir).resolve() tgt = _tgt_dir / resource_path.name tgt = tgt.expanduser() tgt = tgt.resolve() # extract resources extract_resource(src, tgt, is_tutorial) if is_tutorial: set_api_key(tgt)