def __new__( cls, label: str, description: Optional[str] = None, entry_data: Optional["RawMetadataValue"] = None, value: Optional["RawMetadataValue"] = None, ): if description is not None: deprecation_warning( 'The "description" attribute on "MetadataEntry"', "0.15.0", ) value = cast( RawMetadataValue, canonicalize_backcompat_args( new_val=value, new_arg="value", old_val=entry_data, old_arg="entry_data", breaking_version="0.15.0", ), ) value = normalize_metadata_value(value) return super(MetadataEntry, cls).__new__( cls, check.str_param(label, "label"), check.opt_str_param(description, "description"), check.inst_param(value, "value", MetadataValue), )
def output_materialization_config(config_schema=None, required_resource_keys=None, config_cls=None): """Deprecated in favor of dagster_type_materializer""" rename_warning("dagster_type_materializer", "output_materialization_config", "0.10.0") config_schema = canonicalize_backcompat_args( config_schema, "config_schema", config_cls, "config_cls", "0.10.0", ) return dagster_type_materializer(config_schema, required_resource_keys)
def pipeline_execute_command(config, preset, mode, **kwargs): env = ( canonicalize_backcompat_args( (config if config else None), '--config', (kwargs.get('env') if kwargs.get('env') else None), '--env', '0.9.0', stacklevel=2, # this stacklevel can point the warning to this line ) or tuple() # back to default empty tuple ) check.invariant(isinstance(env, tuple)) if preset: if env: raise click.UsageError('Can not use --preset with --env.') return execute_execute_command_with_preset(preset, kwargs, mode) env = list(env) tags = get_tags_from_args(kwargs) result = execute_execute_command(env, kwargs, mode, tags) if not result.success: raise click.ClickException( 'Pipeline run {} resulted in failure.'.format(result.run_id))
def set_intermediate(self, context, dagster_type=None, step_output_handle=None, value=None, runtime_type=None): canonicalize_dagster_type = canonicalize_backcompat_args( dagster_type, 'dagster_type', runtime_type, 'runtime_type', ) # TODO to deprecate in 0.8.0 check.inst_param(context, 'context', SystemPipelineExecutionContext) check.inst_param(canonicalize_dagster_type, 'dagster_type', DagsterType) check.inst_param(step_output_handle, 'step_output_handle', StepOutputHandle) if self.has_intermediate(context, step_output_handle): context.log.warning( 'Replacing existing intermediate for %s.%s' % (step_output_handle.step_key, step_output_handle.output_name)) return self._intermediate_store.set_value( obj=value, context=context, dagster_type=canonicalize_dagster_type, paths=self._get_paths(step_output_handle), )
def __new__(cls, name, dagster_type=None, optional=None, should_materialize=None, runtime_type=None): canonicalize_dagster_type = canonicalize_backcompat_args( new_val=dagster_type, new_arg='dagster_type', old_val=runtime_type, old_arg='runtime_type', ) return super(StepOutput, cls).__new__( cls, name=check.str_param(name, 'name'), optional=check.bool_param(optional, 'optional'), should_materialize=check.bool_param(should_materialize, 'should_materialize'), runtime_type=check.inst_param( canonicalize_dagster_type, 'runtime_type', DagsterType), # TODO to deprecate in 0.8.0 dagster_type=check.inst_param(canonicalize_dagster_type, 'dagster_type', DagsterType), )
def __init__( self, name, cron_schedule, pipeline_name, tags_fn, solid_selection, mode, should_execute, environment_vars, partition_set, environment_dict_fn=None, run_config_fn=None, ): run_config_fn = canonicalize_backcompat_args( run_config_fn, 'run_config_fn', environment_dict_fn, 'environment_dict_fn', '0.9.0' ) super(PartitionScheduleDefinition, self).__init__( name=name, cron_schedule=cron_schedule, pipeline_name=pipeline_name, run_config_fn=run_config_fn, tags_fn=tags_fn, solid_selection=solid_selection, mode=mode, should_execute=should_execute, environment_vars=environment_vars, ) self._partition_set = check.inst_param( partition_set, 'partition_set', PartitionSetDefinition )
def executor(name=None, config_schema=None, required_resource_keys=None, config=None): '''Define an executor. The decorated function should accept an :py:class:`InitExecutorContext` and return an instance of :py:class:`Executor`. Args: name (Optional[str]): The name of the executor. config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in `init_context.executor_config`. required_resource_keys (Optional[Set[str]]): Keys for the resources required by the executor. ''' if callable(name): check.invariant(config_schema is None) check.invariant(required_resource_keys is None) check.invariant(config is None) return _ExecutorDecoratorCallable()(name) return _ExecutorDecoratorCallable( name=name, config_schema=canonicalize_backcompat_args(config_schema, 'config_schema', config, 'config', '0.9.0'), required_resource_keys=required_resource_keys, )
def __init__( self, name, input_defs, compute_fn, output_defs, config=None, description=None, tags=None, required_resource_keys=None, positional_inputs=None, metadata=None, ): self._compute_fn = check.callable_param(compute_fn, 'compute_fn') self._config_field = check_user_facing_opt_config_param( config, 'config') self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, 'required_resource_keys', of_type=str)) super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, 'input_defs', InputDefinition), output_defs=check.list_param(output_defs, 'output_defs', OutputDefinition), description=description, tags=canonicalize_backcompat_args(tags, 'tags', metadata, 'metadata'), positional_inputs=positional_inputs, )
def get_intermediate(self, context, dagster_type=None, step_output_handle=None, runtime_type=None): canonicalize_dagster_type = resolve_dagster_type( canonicalize_backcompat_args( dagster_type, 'dagster_type', runtime_type, 'runtime_type', )) # TODO to deprecate in 0.8.0 check.opt_inst_param(context, 'context', SystemPipelineExecutionContext) check.inst_param(canonicalize_dagster_type, 'dagster_type', DagsterType) check.inst_param(step_output_handle, 'step_output_handle', StepOutputHandle) check.invariant(self.has_intermediate(context, step_output_handle)) return self._intermediate_store.get_value( context=context, dagster_type=canonicalize_dagster_type, paths=self._get_paths(step_output_handle), )
def input_hydration_config(config_schema=None, required_resource_keys=None, config_cls=None): '''Deprecated in favor of dagster_type_loader''' rename_warning('dagster_type_loader', 'input_hydration_config', '0.10.0') config_schema = canonicalize_backcompat_args( config_schema, 'config_schema', config_cls, 'config_cls', '0.10.0', ) return dagster_type_loader(config_schema, required_resource_keys)
def __init__( self, config, default_value=FIELD_NO_DEFAULT_PROVIDED, is_optional=None, is_required=None, description=None, ): from .validate import validate_config from .post_process import post_process_config self.config_type = check.inst(self._resolve_config_arg(config), ConfigType) self.description = check.opt_str_param(description, 'description') check.opt_bool_param(is_optional, 'is_optional') check.opt_bool_param(is_required, 'is_required') canonical_is_required = canonicalize_backcompat_args( new_val=is_required, new_arg='is_required', old_val=is_optional, old_arg='is_optional', coerce_old_to_new=lambda val: not val, additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.', ) if canonical_is_required is True: check.param_invariant( default_value == FIELD_NO_DEFAULT_PROVIDED, 'default_value', 'required arguments should not specify default values', ) self._default_value = default_value # check explicit default value if self.default_provided: # invoke through property in case it is callable value = self.default_value evr = validate_config(self.config_type, value) if not evr.success: raise DagsterInvalidConfigError( 'Invalid default_value for Field.', evr.errors, default_value, ) if canonical_is_required is None: # neither is_required nor is_optional were specified canonical_is_required = not all_optional_type(self.config_type) # on implicitly optional - set the default value # by resolving the defaults of the type if not canonical_is_required and self._default_value == FIELD_NO_DEFAULT_PROVIDED: evr = post_process_config(self.config_type, None) if not evr.success: raise DagsterInvalidConfigError( 'Unable to resolve implicit default_value for Field.', evr.errors, None, ) self._default_value = evr.value self.is_optional = not canonical_is_required
def make_airflow_dag_for_recon_repo( recon_repo, job_name, run_config=None, mode=None, dag_id=None, dag_description=None, dag_kwargs=None, op_kwargs=None, pipeline_name=None, ): job_name = canonicalize_backcompat_args( new_val=job_name, new_arg="job_name", old_val=pipeline_name, old_arg="pipeline_name", breaking_version="future versions", coerce_old_to_new=lambda val: val, ) return _make_airflow_dag( recon_repo=recon_repo, job_name=job_name, run_config=run_config, mode=mode, dag_id=dag_id, dag_description=dag_description, dag_kwargs=dag_kwargs, op_kwargs=op_kwargs, )
def scaffold(module_name, pipeline_name, output_path, config, preset, job_name): """Creates a DAG file for a specified dagster pipeline""" job_name = canonicalize_backcompat_args( new_val=job_name, new_arg="job_name", old_val=pipeline_name, old_arg="pipeline_name", breaking_version="future versions", coerce_old_to_new=lambda val: val, ) check.invariant(job_name is not None, "You must specify either --job-name or --pipeline-name.") check.tuple_param(config, "config", of_type=str) check.invariant(isinstance(config, tuple)) check.invariant( output_path is not None, "You must specify --output-path or set AIRFLOW_HOME to use this script.", ) run_config = construct_environment_yaml(preset, config, job_name, module_name) file_contents = construct_scaffolded_file_contents(module_name, job_name, run_config) # Ensure output_path/dags exists dags_path = os.path.join(os.path.expanduser(output_path), "dags") if not os.path.isdir(dags_path): os.makedirs(dags_path) dag_file = os.path.join(os.path.expanduser(output_path), "dags", job_name + ".py") click.echo("Wrote DAG scaffold to file: %s" % dag_file) with open(dag_file, "wb") as f: f.write(file_contents)
def _wrap(resource_fn): return _ResourceDecoratorCallable( config_schema=canonicalize_backcompat_args(config_schema, 'config_schema', config, 'config', '0.9.0'), description=description, )(resource_fn)
def __new__( cls, name, dagster_type=None, source_type=None, source_handles=None, config_data=None, runtime_type=None, ): canonicalize_dagster_type = canonicalize_backcompat_args( dagster_type, 'dagster_type', runtime_type, 'runtime_type', ) return super(StepInput, cls).__new__( cls, name=check.str_param(name, 'name'), runtime_type=check.inst_param( canonicalize_dagster_type, 'runtime_type', DagsterType ), # TODO to deprecate in 0.8.0 dagster_type=check.inst_param(canonicalize_dagster_type, 'dagster_type', DagsterType), source_type=check.inst_param(source_type, 'source_type', StepInputSourceType), source_handles=check.opt_list_param( source_handles, 'source_handles', of_type=StepOutputHandle ), config_data=config_data, # can be any type )
def __new__( cls, name, is_persistent, required_resource_keys, config_schema=None, system_storage_creation_fn=None, config=None, ): return super(SystemStorageDefinition, cls).__new__( cls, name=check.str_param(name, 'name'), is_persistent=check.bool_param(is_persistent, 'is_persistent'), config_schema=canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ), system_storage_creation_fn=check.opt_callable_param( system_storage_creation_fn, 'system_storage_creation_fn'), required_resource_keys=frozenset( check.set_param(required_resource_keys, 'required_resource_keys', of_type=str)), )
def _wrap(logger_fn): return LoggerDefinition( logger_fn=logger_fn, config_schema=canonicalize_backcompat_args(config_schema, 'config_schema', config, 'config', '0.9.0'), description=description, )
def define_dagstermill_solid( name, notebook_path, input_defs=None, output_defs=None, config=None, required_resource_keys=None, output_notebook=None, config_schema=None, ): '''Wrap a Jupyter notebook in a solid. Arguments: name (str): The name of the solid. notebook_path (str): Path to the backing notebook. input_defs (Optional[List[InputDefinition]]): The solid's inputs. output_defs (Optional[List[OutputDefinition]]): The solid's outputs. Your notebook should call :py:func:`~dagstermill.yield_result` to yield each of these outputs. required_resource_keys (Optional[Set[str]]): The string names of any required resources. output_notebook (Optional[str]): If set, will be used as the name of an injected output of type :py:class:`~dagster.FileHandle` that will point to the executed notebook (in addition to the :py:class:`~dagster.Materialization` that is always created). This respects the :py:class:`~dagster.core.storage.file_manager.FileManager` configured on the pipeline system storage, so, e.g., if :py:class:`~dagster_aws.s3.s3_system_storage` is configured, the output will be a :py:class:`~dagster_aws.s3.S3FileHandle`. Returns: :py:class:`~dagster.SolidDefinition` ''' check.str_param(name, 'name') check.str_param(notebook_path, 'notebook_path') input_defs = check.opt_list_param(input_defs, 'input_defs', of_type=InputDefinition) output_defs = check.opt_list_param(output_defs, 'output_defs', of_type=OutputDefinition) required_resource_keys = check.opt_set_param( required_resource_keys, 'required_resource_keys', of_type=str ) return SolidDefinition( name=name, input_defs=input_defs, compute_fn=_dm_solid_compute(name, notebook_path, output_notebook), output_defs=output_defs + ( [OutputDefinition(dagster_type=FileHandle, name=output_notebook)] if output_notebook else [] ), config_schema=canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ), required_resource_keys=required_resource_keys, description='This solid is backed by the notebook at {path}'.format(path=notebook_path), tags={'notebook_path': notebook_path, 'kind': 'ipynb'}, )
def __new__( cls, kind, key, name, description, display_name, is_builtin, type_param_keys, loader_schema_key=None, materializer_schema_key=None, input_hydration_schema_key=None, output_materialization_schema_key=None, ): return super(DagsterTypeSnap, cls).__new__( cls, kind=check.inst_param(kind, 'kind', DagsterTypeKind), key=check.str_param(key, 'key'), name=check.opt_str_param(name, 'name'), display_name=check.str_param(display_name, 'display_name'), description=check.opt_str_param(description, 'description'), is_builtin=check.bool_param(is_builtin, 'is_builtin'), type_param_keys=check.list_param(type_param_keys, 'type_param_keys', of_type=str), loader_schema_key=canonicalize_backcompat_args( check.opt_str_param(loader_schema_key, 'loader_schema_key'), 'loader_schema_key', check.opt_str_param(input_hydration_schema_key, 'input_hydration_schema_key'), 'input_hydration_schema_key', '0.10.0', ), materializer_schema_key=canonicalize_backcompat_args( check.opt_str_param(materializer_schema_key, 'materializer_schema_key'), 'materializer_schema_key', check.opt_str_param(output_materialization_schema_key, 'output_materialization_schema_key'), 'output_materialization_schema_key', '0.10.0', ), )
def __new__( cls, kind, key, name, description, display_name, is_builtin, type_param_keys, loader_schema_key=None, materializer_schema_key=None, input_hydration_schema_key=None, output_materialization_schema_key=None, ): return super(DagsterTypeSnap, cls).__new__( cls, kind=check.inst_param(kind, "kind", DagsterTypeKind), key=check.str_param(key, "key"), name=check.opt_str_param(name, "name"), display_name=check.str_param(display_name, "display_name"), description=check.opt_str_param(description, "description"), is_builtin=check.bool_param(is_builtin, "is_builtin"), type_param_keys=check.list_param(type_param_keys, "type_param_keys", of_type=str), loader_schema_key=canonicalize_backcompat_args( check.opt_str_param(loader_schema_key, "loader_schema_key"), "loader_schema_key", check.opt_str_param(input_hydration_schema_key, "input_hydration_schema_key"), "input_hydration_schema_key", "0.10.0", ), materializer_schema_key=canonicalize_backcompat_args( check.opt_str_param(materializer_schema_key, "materializer_schema_key"), "materializer_schema_key", check.opt_str_param(output_materialization_schema_key, "output_materialization_schema_key"), "output_materialization_schema_key", "0.10.0", ), )
def _logged_pipeline_launch_command(config, preset, mode, instance, kwargs): check.inst_param(instance, 'instance', DagsterInstance) env = ( canonicalize_backcompat_args( (config if config else None), '--config', (kwargs.get('env') if kwargs.get('env') else None), '--env', '0.9.0', stacklevel=2, # this stacklevel can point the warning to this line ) or tuple() # back to default empty tuple ) env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str)) repo_location = get_repository_location_from_kwargs(kwargs, instance) external_repo = get_external_repository_from_repo_location( repo_location, kwargs.get('repository')) external_pipeline = get_external_pipeline_from_external_repo( external_repo, kwargs.get('pipeline'), ) log_external_repo_stats( instance=instance, external_pipeline=external_pipeline, external_repo=external_repo, source='pipeline_launch_command', ) if preset: if env: raise click.UsageError('Can not use --preset with --config.') preset = external_pipeline.get_preset(preset) else: preset = None run_tags = get_tags_from_args(kwargs) solid_selection = get_solid_selection_from_args(kwargs) pipeline_run = _create_external_pipeline_run( instance=instance, repo_location=repo_location, external_repo=external_repo, external_pipeline=external_pipeline, run_config=get_run_config_from_env_file_list(env), mode=mode, preset=preset, tags=run_tags, solid_selection=solid_selection, ) return instance.launch_run(pipeline_run.run_id, external_pipeline)
def is_new(old_flag=None, new_flag=None): actual_new_flag = canonicalize_backcompat_args( new_val=new_flag, new_arg='new_flag', old_val=old_flag, old_arg='old_flag', coerce_old_to_new=lambda val: not val, additional_warn_txt='Will remove at next release.', ) return actual_new_flag
def is_new(old_flag=None, new_flag=None, include_additional_warn_txt=True): actual_new_flag = canonicalize_backcompat_args( new_val=new_flag, new_arg="new_flag", old_val=old_flag, old_arg="old_flag", breaking_version="0.9.0", coerce_old_to_new=lambda val: not val, additional_warn_txt="Will remove at next release." if include_additional_warn_txt else None, ) return actual_new_flag
def __new__(cls, config_fn, config_schema=None, config=None): return super(ConfigMapping, cls).__new__( cls, config_fn=check.callable_param(config_fn, 'config_fn'), config_schema=canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ), )
def __init__(self, resource_fn, config_schema=None, description=None, config=None): self._resource_fn = check.callable_param(resource_fn, 'resource_fn') self._config_schema = canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ) self._description = check.opt_str_param(description, 'description')
def make_airflow_dag_containerized_for_recon_repo( recon_repo, job_name, image, run_config=None, mode=None, dag_id=None, dag_description=None, dag_kwargs=None, op_kwargs=None, instance=None, pipeline_name=None, ): check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) check.str_param(job_name, "job_name") check.str_param(image, "image") check.opt_dict_param(run_config, "run_config") check.opt_str_param(mode, "mode") check.opt_str_param(dag_id, "dag_id") check.opt_str_param(dag_description, "dag_description") check.opt_dict_param(dag_kwargs, "dag_kwargs") op_kwargs = check.opt_dict_param(op_kwargs, "op_kwargs", key_type=str) check.opt_str_param(pipeline_name, "pipeline_name") op_kwargs["image"] = image job_name = canonicalize_backcompat_args( new_val=job_name, new_arg="job_name", old_val=pipeline_name, old_arg="pipeline_name", breaking_version="future versions", coerce_old_to_new=lambda val: val, ) return _make_airflow_dag( recon_repo=recon_repo, job_name=job_name, run_config=run_config, mode=mode, dag_id=dag_id, dag_description=dag_description, dag_kwargs=dag_kwargs, op_kwargs=op_kwargs, operator=DagsterDockerOperator, instance=instance, )
def __new__( cls, name, pipeline_name, partition_fn, solid_selection=None, mode=None, run_config_fn_for_partition=None, tags_fn_for_partition=lambda _partition: {}, environment_dict_fn_for_partition=None, ): def _wrap(x): if isinstance(x, Partition): return x if isinstance(x, str): return Partition(x) raise DagsterInvalidDefinitionError( 'Expected <Partition> | <str>, received {type}'.format(type=type(x)) ) run_config_fn_for_partition = canonicalize_backcompat_args( run_config_fn_for_partition, 'run_config_fn_for_partition', environment_dict_fn_for_partition, 'environment_dict_fn_for_partition', '0.9.0', ) or (lambda _partition: {}) return super(PartitionSetDefinition, cls).__new__( cls, name=check.str_param(name, 'name'), pipeline_name=check.str_param(pipeline_name, 'pipeline_name'), partition_fn=lambda: [ _wrap(x) for x in check.callable_param(partition_fn, 'partition_fn')() ], solid_selection=check.opt_nullable_list_param( solid_selection, 'solid_selection', of_type=str ), mode=check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME), user_defined_run_config_fn_for_partition=check.callable_param( run_config_fn_for_partition, 'run_config_fn_for_partition' ), user_defined_tags_fn_for_partition=check.callable_param( tags_fn_for_partition, 'tags_fn_for_partition' ), )
def __init__( self, dagster_type=None, name=None, description=None, is_optional=None, is_required=None ): self._name = check_valid_name(check.opt_str_param(name, 'name', DEFAULT_OUTPUT)) self._runtime_type = resolve_dagster_type(dagster_type) self._description = check.opt_str_param(description, 'description') check.opt_bool_param(is_optional, 'is_optional') check.opt_bool_param(is_required, 'is_required') canonical_is_required = canonicalize_backcompat_args( new_val=is_required, new_arg='is_required', old_val=is_optional, old_arg='is_optional', coerce_old_to_new=lambda val: not val, additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.', ) self._optional = False if (canonical_is_required is None) else not canonical_is_required
def get_intermediate(self, context, dagster_type=None, step_output_handle=None, runtime_type=None): canonicalize_dagster_type = canonicalize_backcompat_args( dagster_type, 'dagster_type', runtime_type, 'runtime_type', ) # TODO to deprecate in 0.8.0 check.opt_inst_param(context, 'context', SystemPipelineExecutionContext) check.opt_inst_param(canonicalize_dagster_type, 'dagster_type', DagsterType) check.inst_param(step_output_handle, 'step_output_handle', StepOutputHandle) return self.values[step_output_handle]
def from_files(name, environment_files=None, config_files=None, solid_selection=None, mode=None, tags=None): '''Static constructor for presets from YAML files. Args: name (str): The name of this preset. Must be unique in the presets defined on a given pipeline. config_files (Optional[List[str]]): List of paths or glob patterns for yaml files to load and parse as the environment config for this preset. solid_selection (Optional[List[str]]): A list of solid subselection (including single solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']`` mode (Optional[str]): The mode to apply when executing this preset. (default: 'default') tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset. Returns: PresetDefinition: A PresetDefinition constructed from the provided YAML files. Raises: DagsterInvariantViolationError: When one of the YAML files is invalid and has a parse error. ''' check.str_param(name, 'name') config_files = canonicalize_backcompat_args(config_files, 'config_files', environment_files, 'environment_files', '0.9.0') config_files = check.opt_list_param(config_files, 'config_files') solid_selection = check.opt_nullable_list_param(solid_selection, 'solid_selection', of_type=str) mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME) merged = config_from_files(config_files) return PresetDefinition(name, merged, solid_selection, mode, tags)