def __new__(cls, label=None, description=None, metadata_entries=None, asset_key=None): if asset_key and check.is_str(asset_key): asset_key = AssetKey(parse_asset_key_string(asset_key)) else: check.opt_inst_param(asset_key, 'asset_key', AssetKey) if not label: check.param_invariant( asset_key and asset_key.path, 'label', 'Either label or asset_key with a path must be provided', ) label = asset_key.to_string() return super(Materialization, cls).__new__( cls, label=check.str_param(label, 'label'), description=check.opt_str_param(description, 'description'), metadata_entries=check.opt_list_param(metadata_entries, metadata_entries, of_type=EventMetadataEntry), asset_key=asset_key, )
def __new__( cls, label=None, description=None, metadata_entries=None, asset_key=None, skip_deprecation_warning=False, ): if asset_key and check.is_str(asset_key): asset_key = AssetKey(parse_asset_key_string(asset_key)) else: check.opt_inst_param(asset_key, "asset_key", AssetKey) if not label: check.param_invariant( asset_key and asset_key.path, "label", "Either label or asset_key with a path must be provided", ) label = asset_key.to_string() if not skip_deprecation_warning: warnings.warn( "`Materialization` is deprecated; use `AssetMaterialization` instead." ) return super(Materialization, cls).__new__( cls, label=check.str_param(label, "label"), description=check.opt_str_param(description, "description"), metadata_entries=check.opt_list_param(metadata_entries, metadata_entries, of_type=EventMetadataEntry), asset_key=asset_key, )
def __new__(cls, reconstructor_pointer, reconstructable_args, reconstructable_kwargs): check.inst_param(reconstructor_pointer, "reconstructor_pointer", ModuleCodePointer) # These are lists rather than tuples to circumvent the tuple serdes machinery -- since these # are user-provided, they aren't whitelisted for serdes. check.list_param(reconstructable_args, "reconstructable_args") check.list_param(reconstructable_kwargs, "reconstructable_kwargs") for reconstructable_kwarg in reconstructable_kwargs: check.list_param(reconstructable_kwarg, "reconstructable_kwarg") check.invariant(check.is_str(reconstructable_kwarg[0]), "Bad kwarg key") check.invariant( len(reconstructable_kwarg) == 2, "Bad kwarg of length {length}, should be 2".format( length=len(reconstructable_kwarg)), ) # These are frozenlists, rather than lists, so that they can be hashed and the pointer # stored in the lru_cache on the repository and pipeline get_definition methods reconstructable_args = frozenlist(reconstructable_args) reconstructable_kwargs = frozenlist([ frozenlist(reconstructable_kwarg) for reconstructable_kwarg in reconstructable_kwargs ]) return super(CustomPointer, cls).__new__( cls, reconstructor_pointer, reconstructable_args, reconstructable_kwargs, )
def get_solid_selection_from_args(kwargs): solid_selection_str = kwargs.get("solid_selection") if not check.is_str(solid_selection_str): return None return [ele.strip() for ele in solid_selection_str.split(",") ] if solid_selection_str else None
def validate_tags(tags): valid_tags = {} for key, value in check.opt_dict_param(tags, "tags", key_type=str).items(): if not check.is_str(value): valid = False err_reason = 'Could not JSON encode value "{}"'.format(value) try: str_val = seven.json.dumps(value) err_reason = 'JSON encoding "{json}" of value "{val}" is not equivalent to original value'.format( json=str_val, val=value) valid = seven.json.loads(str_val) == value except Exception: # pylint: disable=broad-except pass if not valid: raise DagsterInvalidDefinitionError( 'Invalid value for tag "{key}", {err_reason}. Tag values must be strings ' "or meet the constraint that json.loads(json.dumps(value)) == value." .format(key=key, err_reason=err_reason)) valid_tags[key] = str_val else: valid_tags[key] = value return frozentags(valid_tags)
def __new__(cls, path=None): if check.is_str(path): path = [validate_asset_key_string(path)] else: path = validate_structured_asset_key( check.list_param(path, 'asset_key_str_or_list', of_type=str)) return super(AssetKey, cls).__new__(cls, path=path)
def __new__(cls, path=None): if check.is_str(path): path = [validate_asset_key_string(path)] elif isinstance(path, list): path = validate_structured_asset_key(check.list_param(path, "path", of_type=str)) else: path = validate_structured_asset_key(check.tuple_param(path, "path", of_type=str)) return super(AssetKey, cls).__new__(cls, path=path)
def __new__(cls, reconstructor_pointer, reconstructable_args, reconstructable_kwargs): check.inst_param(reconstructor_pointer, "reconstructor_pointer", ModuleCodePointer) check.tuple_param(reconstructable_args, "reconstructable_args") check.tuple_param(reconstructable_kwargs, "reconstructable_kwargs") for reconstructable_kwarg in reconstructable_kwargs: check.tuple_param(reconstructable_kwarg, "reconstructable_kwarg") check.invariant(check.is_str(reconstructable_kwarg[0]), "Bad kwarg key") return super(CustomPointer, cls).__new__( cls, reconstructor_pointer, reconstructable_args, reconstructable_kwargs )
def __new__(cls, label, description=None, metadata_entries=None, asset_key=None): if asset_key and check.is_str(asset_key): asset_key = AssetKey(parse_asset_key_string(asset_key)) else: check.opt_inst_param(asset_key, 'asset_key', AssetKey) return super(Materialization, cls).__new__( cls, label=check.str_param(label, 'label'), description=check.opt_str_param(description, 'description'), metadata_entries=check.opt_list_param( metadata_entries, metadata_entries, of_type=EventMetadataEntry ), asset_key=asset_key, )
def __new__(cls, asset_key, description=None, metadata_entries=None, dagster_type=None): if isinstance(asset_key, AssetKey): check.inst_param(asset_key, 'asset_key', AssetKey) elif check.is_str(asset_key): asset_key = AssetKey(parse_asset_key_string(asset_key)) elif isinstance(asset_key, list): check.is_list(asset_key, of_type=str) asset_key = AssetKey(asset_key) else: check.is_tuple(asset_key, of_type=str) asset_key = AssetKey(asset_key) if dagster_type is not None: from dagster.core.types.dagster_type import DagsterType # importing here to resolve circularity dagster_type = check.inst_param(dagster_type, 'dagster_type', DagsterType) metadata_entries = check.opt_list_param(metadata_entries, metadata_entries, of_type=EventMetadataEntry) if metadata_entries is None: metadata_entries = [] metadata_entries.append( EventMetadataEntry.text( (dagster_type.name if dagster_type.name else 'Any'), 'system-type-name')) metadata_entries.append( EventMetadataEntry.text( (dagster_type.description if dagster_type.description else 'Any'), 'system-type-description', )) return super(AssetMaterialization, cls).__new__( cls, asset_key=asset_key, description=check.opt_str_param(description, 'description'), metadata_entries=check.opt_list_param(metadata_entries, metadata_entries, of_type=EventMetadataEntry), )
def __new__(cls, asset_key, description=None, metadata_entries=None): if check.is_str(asset_key): asset_key = AssetKey(parse_asset_key_string(asset_key)) elif isinstance(asset_key, list): check.is_list(asset_key, of_type=str) asset_key = AssetKey(asset_key) else: check.inst_param(asset_key, 'asset_key', AssetKey) return super(AssetMaterialization, cls).__new__( cls, asset_key=asset_key, description=check.opt_str_param(description, 'description'), metadata_entries=check.opt_list_param(metadata_entries, metadata_entries, of_type=EventMetadataEntry), )