def replace(obj, run_id, user_namespace, tags_to_add=None, tags_to_remove=None): _set_current(obj) # While run.replace_tag() can accept 0 additions or 0 removals, we want to encourage # the *obvious* way to achieve their goals. E.g. if they are only adding tags, use "tag add" # over more obscure "tag replace --add ... --add ..." if not tags_to_add and not tags_to_remove: raise CommandException( "Specify at least one tag to add (--add) and one tag to remove (--remove)" ) if not tags_to_remove: raise CommandException( "Specify at least one tag to remove; else please use *tag add*.") if not tags_to_add: raise CommandException( "Specify at least one tag to add, else please use *tag remove*.") user_namespace = resolve_identity( ) if user_namespace is None else user_namespace run = _get_client_run_obj(obj, run_id, user_namespace) run.replace_tags(tags_to_remove, tags_to_add) obj.echo("Operation successful. New tags:") _print_tags_for_one_run(obj, run)
def resolve_task_from_pathspec(flow_name, pathspec): """ resolves a task object for the pathspec query on the CLI. Args: flow_name : (str) : name of flow pathspec (str) : can be `stepname` / `runid/stepname` / `runid/stepname/taskid` Returns: metaflow.Task | None """ from metaflow import Flow, Step, Task from metaflow.exception import MetaflowNotFound # since pathspec can have many variations. pthsplits = pathspec.split("/") task = None run_id = None resolving_from = "task_pathspec" if len(pthsplits) == 1: # This means stepname resolving_from = "stepname" latest_run = Flow(flow_name).latest_run if latest_run is not None: run_id = latest_run.pathspec try: task = latest_run[pathspec].task except KeyError: pass elif len(pthsplits) == 2: # This means runid/stepname namespace(None) resolving_from = "step_pathspec" try: task = Step("/".join([flow_name, pathspec])).task except MetaflowNotFound: pass elif len(pthsplits) == 3: # this means runid/stepname/taskid namespace(None) resolving_from = "task_pathspec" try: task = Task("/".join([flow_name, pathspec])) except MetaflowNotFound: pass else: # raise exception for invalid pathspec format raise CommandException( msg= "The PATHSPEC argument should be of the form 'stepname' Or '<runid>/<stepname>' Or '<runid>/<stepname>/<taskid>'" ) if task is None: # raise Exception that task could not be resolved for the query. raise TaskNotFoundException(pathspec, resolving_from, run_id=run_id) return task
def _execute_cmd(func, flow_name, run_id, user, my_runs, echo): if user and my_runs: raise CommandException("--user and --my-runs are mutually exclusive.") if run_id and my_runs: raise CommandException("--run_id and --my-runs are mutually exclusive.") if my_runs: user = util.get_username() latest_run = True if user and not run_id: latest_run = False if not run_id and latest_run: run_id = util.get_latest_run_id(echo, flow_name) if run_id is None: raise CommandException("A previous run id was not found. Specify --run-id.") func(flow_name, run_id, user, echo)
def _get_client_run_obj(obj, run_id, user_namespace): flow_name = obj.flow.name # handle error messaging for two cases # 1. our user tries to tag a new flow before it is run # 2. our user makes a typo in --namespace try: namespace(user_namespace) Flow(pathspec=flow_name) except MetaflowNotFound: raise CommandException( "No run found for *%s*. Please run the flow before tagging." % flow_name) except MetaflowNamespaceMismatch: raise CommandException( "No run found for *%s* in namespace *%s*. You can switch the namespace using --namespace" % (flow_name, user_namespace)) # throw an error with message to include latest run-id when run_id is None if run_id is None: latest_run_id = Flow(pathspec=flow_name).latest_run.id msg = ("Please specify a run-id using --run-id.\n" "*%s*'s latest run in namespace *%s* has id *%s*." % (flow_name, user_namespace, latest_run_id)) raise CommandException(msg) run_id_parts = run_id.split("/") if len(run_id_parts) == 1: path_spec = "%s/%s" % (flow_name, run_id) else: raise CommandException("Run-id *%s* is not a valid run-id" % run_id) # handle error messaging for three cases # 1. our user makes a typo in --run-id # 2. our user's --run-id does not exist in the default/specified namespace try: namespace(user_namespace) run = Run(pathspec=path_spec) except MetaflowNotFound: raise CommandException("No run *%s* found for flow *%s*" % (path_spec, flow_name)) except MetaflowNamespaceMismatch: msg = "Run *%s* for flow *%s* does not belong to namespace *%s*\n" % ( path_spec, flow_name, user_namespace, ) raise CommandException(msg) return run
def resolve_card( ctx, pathspec, follow_resumed=True, hash=None, type=None, ): """Resolves the card path based on the arguments provided. We allow identifier to be a pathspec or a id of card. Args: ctx: click context object pathspec: pathspec hash (optional): This is to specifically resolve the card via the hash. This is useful when there may be many card with same id or type for a pathspec. type : type of card Raises: CardNotPresentException: No card could be found for the pathspec Returns: (card_paths, card_datastore, taskpathspec) : Tuple[List[str], CardDatastore, str] """ if len(pathspec.split("/")) != 3: raise CommandException( msg="Expecting pathspec of form <runid>/<stepname>/<taskid>" ) flow_name = ctx.obj.flow.name run_id, step_name, task_id = None, None, None # what should be the args we expose run_id, step_name, task_id = pathspec.split("/") pathspec = "/".join([flow_name, pathspec]) # we set namespace to be none to avoid namespace mismatch error. namespace(None) task = Task(pathspec) print_str = "Resolving card: %s" % pathspec if follow_resumed: origin_taskpathspec = resumed_info(task) if origin_taskpathspec: pathspec = origin_taskpathspec ctx.obj.echo( "Resolving card resumed from: %s" % origin_taskpathspec, fg="green", ) else: ctx.obj.echo(print_str, fg="green") else: ctx.obj.echo(print_str, fg="green") # to resolve card_id we first check if the identifier is a pathspec and if it is then we check if the `id` is set or not to resolve card_id # todo : Fix this with `coalesce function` card_paths_found, card_datastore = resolve_paths_from_task( ctx.obj.flow_datastore, pathspec=pathspec, type=type, hash=hash, ) if len(card_paths_found) == 0: # If there are no files found on the Path then raise an error of raise CardNotPresentException( flow_name, run_id, step_name, card_hash=hash, card_type=type, ) return card_paths_found, card_datastore, pathspec
def create( ctx, pathspec, type=None, options=None, timeout=None, render_error_card=False, ): rendered_info = None # Variable holding all the information which will be rendered error_stack_trace = None # Variable which will keep a track of error if len(pathspec.split("/")) != 3: raise CommandException( msg="Expecting pathspec of form <runid>/<stepname>/<taskid>" ) flowname = ctx.obj.flow.name full_pathspec = "/".join([flowname, pathspec]) # todo : Import the changes from Netflix/metaflow#833 for Graph graph_dict = serialize_flowgraph(ctx.obj.graph) task = Task(full_pathspec) from metaflow.plugins import CARDS from metaflow.plugins.cards.exception import CARD_ID_PATTERN from metaflow.cards import ErrorCard error_card = ErrorCard filtered_cards = [CardClass for CardClass in CARDS if CardClass.type == type] card_datastore = CardDatastore(ctx.obj.flow_datastore, pathspec=full_pathspec) if len(filtered_cards) == 0 or type is None: if render_error_card: error_stack_trace = str(CardClassFoundException(type)) else: raise CardClassFoundException(type) if len(filtered_cards) > 0: filtered_card = filtered_cards[0] ctx.obj.echo( "Creating new card of type %s with timeout %s" % (filtered_card.type, timeout), fg="green", ) # If the card is Instatiatable then # first instantiate; If instantiation has a TypeError # then check for render_error_card and accordingly # store the exception as a string or raise the exception try: mf_card = filtered_card(options=options, components=[], graph=graph_dict) except TypeError as e: if render_error_card: mf_card = None error_stack_trace = str(IncorrectCardArgsException(type, options)) else: raise IncorrectCardArgsException(type, options) if mf_card: try: rendered_info = render_card(mf_card, task, timeout_value=timeout) except: if render_error_card: error_stack_trace = str(UnrenderableCardException(type, options)) else: raise UnrenderableCardException(type, options) # if error_stack_trace is not None: rendered_info = error_card().render(task, stack_trace=error_stack_trace) if rendered_info is None and render_error_card: rendered_info = error_card().render( task, stack_trace="No information rendered From card of type %s" % type ) # todo : should we save native type for error card or error type ? if type is not None and re.match(CARD_ID_PATTERN, type) is not None: save_type = type else: save_type = "error" if rendered_info is not None: card_info = card_datastore.save_card(save_type, rendered_info) ctx.obj.echo( "Card created with type: %s and hash: %s" % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]), fg="green", )
def create( ctx, pathspec, type=None, options=None, timeout=None, component_file=None, render_error_card=False, id=None, ): card_id = id rendered_info = None # Variable holding all the information which will be rendered error_stack_trace = None # Variable which will keep a track of error if len(pathspec.split("/")) != 3: raise CommandException( msg="Expecting pathspec of form <runid>/<stepname>/<taskid>" ) flowname = ctx.obj.flow.name full_pathspec = "/".join([flowname, pathspec]) graph_dict, _ = ctx.obj.graph.output_steps() # Components are rendered in a Step and added via `current.card.append` are added here. component_arr = [] if component_file is not None: with open(component_file, "r") as f: component_arr = json.load(f) task = Task(full_pathspec) from metaflow.plugins import CARDS from metaflow.plugins.cards.exception import CARD_ID_PATTERN, TYPE_CHECK_REGEX from metaflow.cards import ErrorCard error_card = ErrorCard filtered_cards = [CardClass for CardClass in CARDS if CardClass.type == type] card_datastore = CardDatastore(ctx.obj.flow_datastore, pathspec=full_pathspec) if len(filtered_cards) == 0 or type is None: if render_error_card: error_stack_trace = str(CardClassFoundException(type)) else: raise CardClassFoundException(type) if len(filtered_cards) > 0: filtered_card = filtered_cards[0] ctx.obj.echo( "Creating new card of type %s with timeout %s" % (filtered_card.type, timeout), fg="green", ) # If the card is Instatiatable then # first instantiate; If instantiation has a TypeError # then check for render_error_card and accordingly # store the exception as a string or raise the exception try: mf_card = filtered_card( options=options, components=component_arr, graph=graph_dict ) except TypeError as e: if render_error_card: mf_card = None error_stack_trace = str(IncorrectCardArgsException(type, options)) else: raise IncorrectCardArgsException(type, options) if mf_card: try: rendered_info = render_card(mf_card, task, timeout_value=timeout) except: if render_error_card: error_stack_trace = str(UnrenderableCardException(type, options)) else: raise UnrenderableCardException(type, options) # if error_stack_trace is not None: rendered_info = error_card().render(task, stack_trace=error_stack_trace) if rendered_info is None and render_error_card: rendered_info = error_card().render( task, stack_trace="No information rendered From card of type %s" % type ) # todo : should we save native type for error card or error type ? if type is not None and re.match(CARD_ID_PATTERN, type) is not None: save_type = type else: save_type = "error" # If card_id is doesn't match regex pattern then we will set it as None if card_id is not None and re.match(CARD_ID_PATTERN, card_id) is None: ctx.obj.echo( "`--id=%s` doesn't match REGEX pattern. `--id` will be set to `None`. Please create `--id` of pattern %s." % (card_id, TYPE_CHECK_REGEX), fg="red", ) card_id = None if rendered_info is not None: card_info = card_datastore.save_card(save_type, rendered_info, card_id=card_id) ctx.obj.echo( "Card created with type: %s and hash: %s" % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]), fg="green", )
def tag_list( obj, run_id, hide_system_tags, list_all, my_runs, group_by_tag, group_by_run, flat, arg_run_id, ): _set_current(obj) if run_id is None and arg_run_id is None and not list_all and not my_runs: # Assume list_all by default list_all = True if list_all and my_runs: raise CommandException( "Option --all cannot be used together with --my-runs.") if run_id is not None and arg_run_id is not None: raise CommandException( "Specify a run either using --run-id or as an argument but not both" ) if arg_run_id is not None: run_id = arg_run_id if group_by_run and group_by_tag: raise CommandException( "Option --group-by-tag cannot be used with --group-by-run") if flat and (group_by_run or group_by_tag): raise CommandException( "Option --flat cannot be used with any --group-by-* option") system_tags_by_some_grouping = dict() all_tags_by_some_grouping = dict() def _populate_tag_groups_from_run(_run): if group_by_run: if hide_system_tags: all_tags_by_some_grouping[ _run.pathspec] = _run.tags - _run.system_tags else: system_tags_by_some_grouping[_run.pathspec] = _run.system_tags all_tags_by_some_grouping[_run.pathspec] = _run.tags elif group_by_tag: for t in _run.tags - _run.system_tags: all_tags_by_some_grouping.setdefault(t, []).append(_run.pathspec) if not hide_system_tags: for t in _run.system_tags: system_tags_by_some_grouping.setdefault(t, []).append( _run.pathspec) else: if hide_system_tags: all_tags_by_some_grouping.setdefault("_", set()).update( _run.tags.difference(_run.system_tags)) else: system_tags_by_some_grouping.setdefault("_", set()).update( _run.system_tags) all_tags_by_some_grouping.setdefault("_", set()).update(_run.tags) pathspecs = [] if list_all or my_runs: user_namespace = resolve_identity() if my_runs else None namespace(user_namespace) try: flow = Flow(pathspec=obj.flow.name) except MetaflowNotFound: raise CommandException( "Cannot list tags because the flow %s has never been run." % (obj.flow.name, )) for run in flow.runs(): _populate_tag_groups_from_run(run) pathspecs.append(run.pathspec) else: run = _get_client_run_obj(obj, run_id, None) _populate_tag_groups_from_run(run) pathspecs.append(run.pathspec) if not group_by_run and not group_by_tag: # We list all the runs that match to print them out if needed. system_tags_by_some_grouping[",".join( pathspecs)] = system_tags_by_some_grouping.get("_", set()) all_tags_by_some_grouping[",".join( pathspecs)] = all_tags_by_some_grouping.get("_", set()) if "_" in system_tags_by_some_grouping: del system_tags_by_some_grouping["_"] if "_" in all_tags_by_some_grouping: del all_tags_by_some_grouping["_"] if flat: if len(all_tags_by_some_grouping) != 1: raise MetaflowInternalError("Failed to flatten tag set") for v in all_tags_by_some_grouping.values(): for tag in v: obj.echo(tag) return _print_tags_for_runs_by_groups(obj, system_tags_by_some_grouping, all_tags_by_some_grouping, group_by_tag)