def compress_list(lst, separator=',', rangedelim=':', zlibmarker='!', zlibmin=500): bad_items = [ x for x in lst if separator in x or rangedelim in x or zlibmarker in x ] if bad_items: raise MetaflowInternalError("Item '%s' includes a delimiter character " "so it can't be compressed" % bad_items[0]) # Three output modes: lcp = longest_common_prefix(lst) if len(lst) < 2 or not lcp: # 1. Just a comma-separated list res = separator.join(lst) else: # 2. Prefix and a comma-separated list of suffixes lcplen = len(lcp) residuals = [e[lcplen:] for e in lst] res = rangedelim.join((lcp, separator.join(residuals))) if len(res) < zlibmin: return res else: # 3. zlib-compressed, base64-encoded, prefix-encoded list # interestingly, a typical zlib-encoded list of suffixes # has plenty of redundancy. Decoding the data *twice* helps a # lot compressed = zlib.compress(zlib.compress(to_bytes(res))) return zlibmarker + base64.b64encode(compressed).decode('utf-8')
def _deduce_run_id_from_meta_dir(meta_dir_path, sub_type): curr_order = ObjectOrder.type_to_order(sub_type) levels_to_ascend = curr_order - ObjectOrder.type_to_order("run") if levels_to_ascend < 0: return None curr_path = meta_dir_path for _ in range(levels_to_ascend + 1): # +1 to account for ../_meta curr_path, _ = os.path.split(curr_path) _, run_id = os.path.split(curr_path) if not run_id: raise MetaflowInternalError( "Failed to deduce run_id from meta dir %s" % meta_dir_path) return run_id
def _persist_tags_for_run(flow_id, run_id, tags, system_tags): subpath = LocalMetadataProvider._create_and_get_metadir( flow_name=flow_id, run_id=run_id) selfname = os.path.join(subpath, "_self.json") if not os.path.isfile(selfname): raise MetaflowInternalError( msg="Could not verify Run existence on disk - missing %s" % selfname) LocalMetadataProvider._save_meta( subpath, { "_self": MetadataProvider._run_to_json_static( flow_id, run_id=run_id, tags=tags, sys_tags=system_tags) }, allow_overwrite=True, )
def __init__(self, pathspec=None, _object=None, _parent=None, _namespace_check=True): self._metaflow = Metaflow() self._parent = _parent self._path_components = None if pathspec: ids = pathspec.split('/') self.id = ids[-1] self._pathspec = pathspec self._object = self._get_object(*ids) else: self._object = _object self._pathspec = pathspec if self._NAME in ('flow', 'task'): self.id = str(self._object[self._NAME + '_id']) elif self._NAME == 'run': self.id = str(self._object['run_number']) elif self._NAME == 'step': self.id = str(self._object['step_name']) elif self._NAME == 'artifact': self.id = str(self._object['name']) else: raise MetaflowInternalError(msg="Unknown type: %s" % self._NAME) self._created_at = time.strftime( '%Y-%m-%dT%H:%M:%SZ', time.gmtime(self._object['ts_epoch'] // 1000)) self._tags = frozenset( chain( self._object.get('system_tags') or [], self._object.get('tags') or [])) if _namespace_check and not self.is_in_namespace(): raise MetaflowNamespaceMismatch(current_namespace)
def loglines(self, stream, as_unicode=True): """ Return an iterator over (utc_timestamp, logline) tuples. If as_unicode=False, logline is returned as a byte object. Otherwise, it is returned as a (unicode) string. """ from metaflow.mflog.mflog import merge_logs from metaflow.mflog import LOG_SOURCES from metaflow.datastore import DATASTORES ds_type = self.metadata_dict.get('ds-type') ds_root = self.metadata_dict.get('ds-root') ds_cls = DATASTORES.get(ds_type, None) if ds_cls is None: raise MetaflowInternalError('Datastore %s was not found' % ds_type) ds_cls.datastore_root = ds_root # It is possible that a task fails before any metadata has been # recorded. In this case, we assume that we are executing the # first attempt. # # FIXME: Technically we are looking at the latest *recorded* attempt # here. It is possible that logs exists for a newer attempt that # just failed to record metadata. We could make this logic more robust # and guarantee that we always return the latest available log. ds = ds_cls(self._object['flow_id'], run_id=str(self._object['run_number']), step_name=self._object['step_name'], task_id=str(self._object['task_id']), mode='r', attempt=int(self.metadata_dict.get('attempt', 0)), allow_unsuccessful=True) logs = ds.load_logs(LOG_SOURCES, stream) for line in merge_logs([blob for _, blob in logs]): msg = to_unicode(line.msg) if as_unicode else line.msg yield line.utc_tstamp, msg
def get_object(cls, obj_type, sub_type, filters=None, *args): '''Returns the requested object depending on obj_type and sub_type obj_type can be one of 'root', 'flow', 'run', 'step', 'task', or 'artifact' sub_type describes the aggregation required and can be either: 'metadata', 'self' or any of obj_type provided that it is slotted below the object itself. For example, if obj_type is 'flow', you can specify 'run' to get all the runs in that flow. A few special rules: - 'metadata' is only allowed for obj_type 'task' - For obj_type 'artifact', only 'self' is allowed A few examples: - To get a list of all flows: - set obj_type to 'root' and sub_type to 'flow' - To get a list of all tasks: - set obj_type to 'root' and sub_type to 'task' - To get a list of all artifacts in a task: - set obj_type to 'task' and sub_type to 'artifact' - To get information about a specific flow: - set obj_type to 'flow' and sub_type to 'self' Parameters ---------- obj_type : string One of 'root', 'flow', 'run', 'step', 'task', 'artifact' or 'metadata' sub_type : string Same as obj_type with the addition of 'self' filters : dict Dictionary with keys 'any_tags', 'tags' and 'system_tags'. If specified will return only objects that have the specified tags present. Filters are ANDed together so all tags must be present for the object to be returned. Return ------ object or list : Depending on the call, the type of object return varies ''' obj_order = { 'root': 0, 'flow': 1, 'run': 2, 'step': 3, 'task': 4, 'artifact': 5, 'metadata': 6, 'self': 7 } type_order = obj_order.get(obj_type) sub_order = obj_order.get(sub_type) if type_order is None: raise MetaflowInternalError(msg='Cannot find type %s' % obj_type) if type_order > 5: raise MetaflowInternalError(msg='Type %s is not allowed' % obj_type) if sub_order is None: raise MetaflowInternalError(msg='Cannot find subtype %s' % sub_type) if type_order >= sub_order: raise MetaflowInternalError(msg='Subtype %s not allowed for %s' % (sub_type, obj_type)) # Metadata is always only at the task level if sub_type == 'metadata' and obj_type != 'task': raise MetaflowInternalError( msg='Metadata can only be retrieved at the task level') return cls._get_object_internal(obj_type, type_order, sub_type, sub_order, filters, *args)
def resolve_token( name, token_prefix, obj, authorize, given_token, generate_new_token, is_project ): # 1) retrieve the previous deployment, if one exists workflow = StepFunctions.get_existing_deployment(name) if workflow is None: obj.echo( "It seems this is the first time you are deploying *%s* to " "AWS Step Functions." % name ) prev_token = None else: prev_user, prev_token = workflow # 2) authorize this deployment if prev_token is not None: if authorize is None: authorize = load_token(token_prefix) elif authorize.startswith("production:"): authorize = authorize[11:] # we allow the user who deployed the previous version to re-deploy, # even if they don't have the token if prev_user != get_username() and authorize != prev_token: obj.echo( "There is an existing version of *%s* on AWS Step " "Functions which was deployed by the user " "*%s*." % (name, prev_user) ) obj.echo( "To deploy a new version of this flow, you need to use " "the same production token that they used. " ) obj.echo( "Please reach out to them to get the token. Once you " "have it, call this command:" ) obj.echo(" step-functions create --authorize MY_TOKEN", fg="green") obj.echo( 'See "Organizing Results" at docs.metaflow.org for more ' "information about production tokens." ) raise IncorrectProductionToken( "Try again with the correct " "production token." ) # 3) do we need a new token or should we use the existing token? if given_token: if is_project: # we rely on a known prefix for @project tokens, so we can't # allow the user to specify a custom token with an arbitrary prefix raise MetaflowException( "--new-token is not supported for " "@projects. Use --generate-new-token to " "create a new token." ) if given_token.startswith("production:"): given_token = given_token[11:] token = given_token obj.echo("") obj.echo("Using the given token, *%s*." % token) elif prev_token is None or generate_new_token: token = new_token(token_prefix, prev_token) if token is None: if prev_token is None: raise MetaflowInternalError( "We could not generate a new " "token. This is unexpected. " ) else: raise MetaflowException( "--generate-new-token option is not " "supported after using --new-token. " "Use --new-token to make a new " "namespace." ) obj.echo("") obj.echo("A new production token generated.") else: token = prev_token obj.echo("") obj.echo("The namespace of this production flow is") obj.echo(" production:%s" % token, fg="green") obj.echo( "To analyze results of this production flow " "add this line in your notebooks:" ) obj.echo(' namespace("production:%s")' % token, fg="green") obj.echo( "If you want to authorize other people to deploy new versions " "of this flow to AWS Step Functions, they need to call" ) obj.echo(" step-functions create --authorize %s" % token, fg="green") obj.echo("when deploying this flow to AWS Step Functions for the first " "time.") obj.echo( 'See "Organizing Results" at https://docs.metaflow.org/ for more ' "information about production tokens." ) obj.echo("") store_token(token_prefix, token) return token
def _request( cls, monitor, path, method, data=None, retry_409_path=None, return_raw_resp=False, ): if cls.INFO is None: raise MetaflowException( "Missing Metaflow Service URL. " "Specify with METAFLOW_SERVICE_URL environment variable" ) supported_methods = ("GET", "PATCH", "POST") if method not in supported_methods: raise MetaflowException( "Only these methods are supported: %s, but got %s" % (supported_methods, method) ) url = os.path.join(cls.INFO, path.lstrip("/")) for i in range(METADATA_SERVICE_NUM_RETRIES): try: if method == "GET": if monitor: with monitor.measure("metaflow.service_metadata.get"): resp = requests.get(url, headers=METADATA_SERVICE_HEADERS) else: resp = requests.get(url, headers=METADATA_SERVICE_HEADERS) elif method == "POST": if monitor: with monitor.measure("metaflow.service_metadata.post"): resp = requests.post( url, headers=METADATA_SERVICE_HEADERS, json=data ) else: resp = requests.post( url, headers=METADATA_SERVICE_HEADERS, json=data ) elif method == "PATCH": if monitor: with monitor.measure("metaflow.service_metadata.patch"): resp = requests.patch( url, headers=METADATA_SERVICE_HEADERS, json=data ) else: resp = requests.patch( url, headers=METADATA_SERVICE_HEADERS, json=data ) else: raise MetaflowInternalError("Unexpected HTTP method %s" % (method,)) except MetaflowInternalError: raise except: # noqa E722 if monitor: with monitor.count("metaflow.service_metadata.failed_request"): if i == METADATA_SERVICE_NUM_RETRIES - 1: raise else: if i == METADATA_SERVICE_NUM_RETRIES - 1: raise resp = None else: if return_raw_resp: return resp, True if resp.status_code < 300: return resp.json(), True elif resp.status_code == 409 and data is not None: # a special case: the post fails due to a conflict # this could occur when we missed a success response # from the first POST request but the request # actually went though, so a subsequent POST # returns 409 (conflict) or we end up with a # conflict while running on AWS Step Functions # instead of retrying the post we retry with a get since # the record is guaranteed to exist if retry_409_path: v, _ = cls._request(monitor, retry_409_path, "GET") return v, False else: return None, False elif resp.status_code != 503: raise ServiceException( "Metadata request (%s) failed (code %s): %s" % (path, resp.status_code, resp.text), resp.status_code, resp.text, ) time.sleep(2 ** i) if resp: raise ServiceException( "Metadata request (%s) failed (code %s): %s" % (path, resp.status_code, resp.text), resp.status_code, resp.text, ) else: raise ServiceException("Metadata request (%s) failed" % path)
def get_object(cls, obj_type, sub_type, filters, attempt, *args): """Returns the requested object depending on obj_type and sub_type obj_type can be one of 'root', 'flow', 'run', 'step', 'task', or 'artifact' sub_type describes the aggregation required and can be either: 'metadata', 'self' or any of obj_type provided that it is slotted below the object itself. For example, if obj_type is 'flow', you can specify 'run' to get all the runs in that flow. A few special rules: - 'metadata' is only allowed for obj_type 'task' - For obj_type 'artifact', only 'self' is allowed A few examples: - To get a list of all flows: - set obj_type to 'root' and sub_type to 'flow' - To get a list of all tasks: - set obj_type to 'root' and sub_type to 'task' - To get a list of all artifacts in a task: - set obj_type to 'task' and sub_type to 'artifact' - To get information about a specific flow: - set obj_type to 'flow' and sub_type to 'self' Parameters ---------- obj_type : string One of 'root', 'flow', 'run', 'step', 'task', 'artifact' or 'metadata' sub_type : string Same as obj_type with the addition of 'self' filters : dict Dictionary with keys 'any_tags', 'tags' and 'system_tags'. If specified will return only objects that have the specified tags present. Filters are ANDed together so all tags must be present for the object to be returned. attempt : int or None If None, for metadata and artifacts: - returns information about the latest attempt for artifacts - returns all metadata across all attempts Otherwise, returns information about metadata and artifacts for that attempt only. NOTE: For older versions of Metaflow (pre 2.4.0), the attempt for metadata is not known; in that case, all metadata is returned (as if None was passed in). Return ------ object or list : Depending on the call, the type of object return varies """ type_order = ObjectOrder.type_to_order(obj_type) sub_order = ObjectOrder.type_to_order(sub_type) if type_order is None: raise MetaflowInternalError(msg="Cannot find type %s" % obj_type) if type_order >= ObjectOrder.type_to_order("metadata"): raise MetaflowInternalError(msg="Type %s is not allowed" % obj_type) if sub_order is None: raise MetaflowInternalError(msg="Cannot find subtype %s" % sub_type) if type_order >= sub_order: raise MetaflowInternalError(msg="Subtype %s not allowed for %s" % (sub_type, obj_type)) # Metadata is always only at the task level if sub_type == "metadata" and obj_type != "task": raise MetaflowInternalError( msg="Metadata can only be retrieved at the task level") if attempt is not None: try: attempt_int = int(attempt) if attempt_int < 0: raise ValueError("Attempt can only be positive") except ValueError: raise ValueError("Attempt can only be a positive integer") else: attempt_int = None pre_filter = cls._get_object_internal(obj_type, type_order, sub_type, sub_order, filters, attempt_int, *args) if attempt_int is None or sub_type != "metadata": # If no attempt or not for metadata, just return as is return pre_filter return MetadataProvider._reconstruct_metadata_for_attempt( pre_filter, attempt_int)
def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): # This is guaranteed by MetaflowProvider.get_object(), sole intended caller if obj_type in ("metadata", "self"): raise MetaflowInternalError(msg="Type %s is not allowed" % obj_type) if obj_type not in ("root", "flow", "run", "step", "task", "artifact"): raise MetaflowInternalError(msg="Unexpected object type %s" % obj_type) from metaflow.datastore.local_storage import LocalStorage if obj_type == "artifact": # Artifacts are actually part of the tasks in the filesystem # E.g. we get here for (obj_type, sub_type) == (artifact, self) obj_type = "task" sub_type = "artifact" sub_order = obj_order obj_order = obj_order - 1 if obj_type != ObjectOrder.order_to_type(obj_order): raise MetaflowInternalError( "Object type order mismatch %s %s" % (obj_type, ObjectOrder.order_to_type(obj_order))) if sub_type != ObjectOrder.order_to_type(sub_order): raise MetaflowInternalError( "Sub type order mismatch %s %s" % (sub_type, ObjectOrder.order_to_type(sub_order))) RUN_ORDER = ObjectOrder.type_to_order("run") if obj_type not in ("root", "flow", "run", "step", "task"): raise MetaflowInternalError(msg="Unexpected object type %s" % obj_type) # Special handling of self, artifact, and metadata if sub_type == "self": meta_path = LocalMetadataProvider._get_metadir(*args[:obj_order]) if meta_path is None: return None self_file = os.path.join(meta_path, "_self.json") if os.path.isfile(self_file): obj = MetadataProvider._apply_filter( [LocalMetadataProvider._read_json_file(self_file)], filters)[0] # For non-descendants of a run, we are done if obj_order <= RUN_ORDER: return obj if obj_type not in ("step", "task"): raise MetaflowInternalError( msg="Unexpected object type %s" % obj_type) run = LocalMetadataProvider.get_object( "run", "self", {}, None, *args[:RUN_ORDER] # *[flow_id, run_id] ) if not run: raise MetaflowInternalError(msg="Could not find run %s" % str(args[:RUN_ORDER])) obj["tags"] = run.get("tags", []) obj["system_tags"] = run.get("system_tags", []) return obj return None if sub_type == "artifact": if obj_type not in ("root", "flow", "run", "step", "task"): raise MetaflowInternalError(msg="Unexpected object type %s" % obj_type) meta_path = LocalMetadataProvider._get_metadir(*args[:obj_order]) result = [] if meta_path is None: return result successful_attempt = attempt if successful_attempt is None: attempt_done_files = os.path.join(meta_path, "sysmeta_attempt-done_*") attempts_done = sorted(glob.iglob(attempt_done_files)) if attempts_done: successful_attempt = int( LocalMetadataProvider._read_json_file( attempts_done[-1])["value"]) if successful_attempt is not None: which_artifact = "*" if len(args) >= sub_order: which_artifact = args[sub_order - 1] artifact_files = os.path.join( meta_path, "%d_artifact_%s.json" % (successful_attempt, which_artifact), ) for obj in glob.iglob(artifact_files): result.append(LocalMetadataProvider._read_json_file(obj)) # We are getting artifacts. We should overlay with ancestral run's tags run = LocalMetadataProvider.get_object( "run", "self", {}, None, *args[:RUN_ORDER] # *[flow_id, run_id] ) if not run: raise MetaflowInternalError(msg="Could not find run %s" % str(args[:RUN_ORDER])) for obj in result: obj["tags"] = run.get("tags", []) obj["system_tags"] = run.get("system_tags", []) if len(result) == 1: return result[0] return result if sub_type == "metadata": # artifact is not expected because if obj_type=artifact on function entry, we transform to =task if obj_type not in ("root", "flow", "run", "step", "task"): raise MetaflowInternalError(msg="Unexpected object type %s" % obj_type) result = [] meta_path = LocalMetadataProvider._get_metadir(*args[:obj_order]) if meta_path is None: return result files = os.path.join(meta_path, "sysmeta_*") for obj in glob.iglob(files): result.append(LocalMetadataProvider._read_json_file(obj)) return result # For the other types, we locate all the objects we need to find and return them if obj_type not in ("root", "flow", "run", "step", "task"): raise MetaflowInternalError(msg="Unexpected object type %s" % obj_type) if sub_type not in ("flow", "run", "step", "task"): raise MetaflowInternalError(msg="unexpected sub type %s" % sub_type) obj_path = LocalMetadataProvider._make_path(*args[:obj_order], create_on_absent=False) result = [] if obj_path is None: return result skip_dirs = "*/" * (sub_order - obj_order) all_meta = os.path.join(obj_path, skip_dirs, LocalStorage.METADATA_DIR) SelfInfo = collections.namedtuple("SelfInfo", ["filepath", "run_id"]) self_infos = [] for meta_path in glob.iglob(all_meta): self_file = os.path.join(meta_path, "_self.json") if not os.path.isfile(self_file): continue run_id = None # flow and run do not need info from ancestral run if sub_type in ("step", "task"): run_id = LocalMetadataProvider._deduce_run_id_from_meta_dir( meta_path, sub_type) # obj_type IS run, or more granular than run, let's do sanity check vs args if obj_order >= RUN_ORDER: if run_id != args[RUN_ORDER - 1]: raise MetaflowInternalError( msg="Unexpected run id %s deduced from meta path" % run_id) self_infos.append(SelfInfo(filepath=self_file, run_id=run_id)) for self_info in self_infos: obj = LocalMetadataProvider._read_json_file(self_info.filepath) if self_info.run_id: flow_id_from_args = args[0] run = LocalMetadataProvider.get_object( "run", "self", {}, None, flow_id_from_args, self_info.run_id, ) if not run: raise MetaflowInternalError( msg="Could not find run %s, %s" % (flow_id_from_args, self_info.run_id)) obj["tags"] = run.get("tags", []) obj["system_tags"] = run.get("system_tags", []) result.append(obj) return MetadataProvider._apply_filter(result, filters)
def tag_list( obj, run_id, hide_system_tags, list_all, my_runs, group_by_tag, group_by_run, flat, arg_run_id, ): _set_current(obj) if run_id is None and arg_run_id is None and not list_all and not my_runs: # Assume list_all by default list_all = True if list_all and my_runs: raise CommandException( "Option --all cannot be used together with --my-runs.") if run_id is not None and arg_run_id is not None: raise CommandException( "Specify a run either using --run-id or as an argument but not both" ) if arg_run_id is not None: run_id = arg_run_id if group_by_run and group_by_tag: raise CommandException( "Option --group-by-tag cannot be used with --group-by-run") if flat and (group_by_run or group_by_tag): raise CommandException( "Option --flat cannot be used with any --group-by-* option") system_tags_by_some_grouping = dict() all_tags_by_some_grouping = dict() def _populate_tag_groups_from_run(_run): if group_by_run: if hide_system_tags: all_tags_by_some_grouping[ _run.pathspec] = _run.tags - _run.system_tags else: system_tags_by_some_grouping[_run.pathspec] = _run.system_tags all_tags_by_some_grouping[_run.pathspec] = _run.tags elif group_by_tag: for t in _run.tags - _run.system_tags: all_tags_by_some_grouping.setdefault(t, []).append(_run.pathspec) if not hide_system_tags: for t in _run.system_tags: system_tags_by_some_grouping.setdefault(t, []).append( _run.pathspec) else: if hide_system_tags: all_tags_by_some_grouping.setdefault("_", set()).update( _run.tags.difference(_run.system_tags)) else: system_tags_by_some_grouping.setdefault("_", set()).update( _run.system_tags) all_tags_by_some_grouping.setdefault("_", set()).update(_run.tags) pathspecs = [] if list_all or my_runs: user_namespace = resolve_identity() if my_runs else None namespace(user_namespace) try: flow = Flow(pathspec=obj.flow.name) except MetaflowNotFound: raise CommandException( "Cannot list tags because the flow %s has never been run." % (obj.flow.name, )) for run in flow.runs(): _populate_tag_groups_from_run(run) pathspecs.append(run.pathspec) else: run = _get_client_run_obj(obj, run_id, None) _populate_tag_groups_from_run(run) pathspecs.append(run.pathspec) if not group_by_run and not group_by_tag: # We list all the runs that match to print them out if needed. system_tags_by_some_grouping[",".join( pathspecs)] = system_tags_by_some_grouping.get("_", set()) all_tags_by_some_grouping[",".join( pathspecs)] = all_tags_by_some_grouping.get("_", set()) if "_" in system_tags_by_some_grouping: del system_tags_by_some_grouping["_"] if "_" in all_tags_by_some_grouping: del all_tags_by_some_grouping["_"] if flat: if len(all_tags_by_some_grouping) != 1: raise MetaflowInternalError("Failed to flatten tag set") for v in all_tags_by_some_grouping.values(): for tag in v: obj.echo(tag) return _print_tags_for_runs_by_groups(obj, system_tags_by_some_grouping, all_tags_by_some_grouping, group_by_tag)
def __init__( self, pathspec=None, attempt=None, _object=None, _parent=None, _namespace_check=True, ): self._metaflow = Metaflow() self._parent = _parent self._path_components = None self._attempt = attempt if self._attempt is not None: if self._NAME not in ["task", "artifact"]: raise MetaflowNotFound( "Attempts can only be specified for Task or DataArtifact") try: self._attempt = int(self._attempt) except ValueError: raise MetaflowNotFound("Attempt can only be an integer") if self._attempt < 0: raise MetaflowNotFound("Attempt can only be non-negative") elif self._attempt >= MAX_ATTEMPTS: raise MetaflowNotFound("Attempt can only be smaller than %d" % MAX_ATTEMPTS) # NOTE: It is possible that no attempt exists but we can't # distinguish between "attempt will happen" and "no such # attempt exists". if pathspec: ids = pathspec.split("/") self.id = ids[-1] self._pathspec = pathspec self._object = self._get_object(*ids) else: self._object = _object self._pathspec = pathspec if self._NAME in ("flow", "task"): self.id = str(self._object[self._NAME + "_id"]) elif self._NAME == "run": self.id = str(self._object["run_number"]) elif self._NAME == "step": self.id = str(self._object["step_name"]) elif self._NAME == "artifact": self.id = str(self._object["name"]) else: raise MetaflowInternalError(msg="Unknown type: %s" % self._NAME) self._created_at = datetime.fromtimestamp(self._object["ts_epoch"] / 1000.0) self._tags = frozenset( chain( self._object.get("system_tags") or [], self._object.get("tags") or [])) if _namespace_check and not self.is_in_namespace(): raise MetaflowNamespaceMismatch(current_namespace)