def __new__(
cls,
label: str,
description: Optional[str] = None,
entry_data: Optional["RawMetadataValue"] = None,
value: Optional["RawMetadataValue"] = None,
):
if description is not None:
deprecation_warning(
'The "description" attribute on "MetadataEntry"',
"0.15.0",
)
value = cast(
RawMetadataValue,
canonicalize_backcompat_args(
new_val=value,
new_arg="value",
old_val=entry_data,
old_arg="entry_data",
breaking_version="0.15.0",
),
)
value = normalize_metadata_value(value)
return super(MetadataEntry, cls).__new__(
cls,
check.str_param(label, "label"),
check.opt_str_param(description, "description"),
check.inst_param(value, "value", MetadataValue),
)
def output_materialization_config(config_schema=None, required_resource_keys=None, config_cls=None):
"""Deprecated in favor of dagster_type_materializer"""
rename_warning("dagster_type_materializer", "output_materialization_config", "0.10.0")
config_schema = canonicalize_backcompat_args(
config_schema, "config_schema", config_cls, "config_cls", "0.10.0",
)
return dagster_type_materializer(config_schema, required_resource_keys)
def pipeline_execute_command(config, preset, mode, **kwargs):
env = (
canonicalize_backcompat_args(
(config if config else None),
'--config',
(kwargs.get('env') if kwargs.get('env') else None),
'--env',
'0.9.0',
stacklevel=2, # this stacklevel can point the warning to this line
) or tuple() # back to default empty tuple
)
check.invariant(isinstance(env, tuple))
if preset:
if env:
raise click.UsageError('Can not use --preset with --env.')
return execute_execute_command_with_preset(preset, kwargs, mode)
env = list(env)
tags = get_tags_from_args(kwargs)
result = execute_execute_command(env, kwargs, mode, tags)
if not result.success:
raise click.ClickException(
'Pipeline run {} resulted in failure.'.format(result.run_id))
def set_intermediate(self,
context,
dagster_type=None,
step_output_handle=None,
value=None,
runtime_type=None):
canonicalize_dagster_type = canonicalize_backcompat_args(
dagster_type,
'dagster_type',
runtime_type,
'runtime_type',
) # TODO to deprecate in 0.8.0
check.inst_param(context, 'context', SystemPipelineExecutionContext)
check.inst_param(canonicalize_dagster_type, 'dagster_type',
DagsterType)
check.inst_param(step_output_handle, 'step_output_handle',
StepOutputHandle)
if self.has_intermediate(context, step_output_handle):
context.log.warning(
'Replacing existing intermediate for %s.%s' %
(step_output_handle.step_key, step_output_handle.output_name))
return self._intermediate_store.set_value(
obj=value,
context=context,
dagster_type=canonicalize_dagster_type,
paths=self._get_paths(step_output_handle),
)
def __new__(cls,
name,
dagster_type=None,
optional=None,
should_materialize=None,
runtime_type=None):
canonicalize_dagster_type = canonicalize_backcompat_args(
new_val=dagster_type,
new_arg='dagster_type',
old_val=runtime_type,
old_arg='runtime_type',
)
return super(StepOutput, cls).__new__(
cls,
name=check.str_param(name, 'name'),
optional=check.bool_param(optional, 'optional'),
should_materialize=check.bool_param(should_materialize,
'should_materialize'),
runtime_type=check.inst_param(
canonicalize_dagster_type, 'runtime_type',
DagsterType), # TODO to deprecate in 0.8.0
dagster_type=check.inst_param(canonicalize_dagster_type,
'dagster_type', DagsterType),
)
def __init__(
self,
name,
cron_schedule,
pipeline_name,
tags_fn,
solid_selection,
mode,
should_execute,
environment_vars,
partition_set,
environment_dict_fn=None,
run_config_fn=None,
):
run_config_fn = canonicalize_backcompat_args(
run_config_fn, 'run_config_fn', environment_dict_fn, 'environment_dict_fn', '0.9.0'
)
super(PartitionScheduleDefinition, self).__init__(
name=name,
cron_schedule=cron_schedule,
pipeline_name=pipeline_name,
run_config_fn=run_config_fn,
tags_fn=tags_fn,
solid_selection=solid_selection,
mode=mode,
should_execute=should_execute,
environment_vars=environment_vars,
)
self._partition_set = check.inst_param(
partition_set, 'partition_set', PartitionSetDefinition
)
def executor(name=None,
config_schema=None,
required_resource_keys=None,
config=None):
'''Define an executor.
The decorated function should accept an :py:class:`InitExecutorContext` and return an instance
of :py:class:`Executor`.
Args:
name (Optional[str]): The name of the executor.
config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in
`init_context.executor_config`.
required_resource_keys (Optional[Set[str]]): Keys for the resources required by the
executor.
'''
if callable(name):
check.invariant(config_schema is None)
check.invariant(required_resource_keys is None)
check.invariant(config is None)
return _ExecutorDecoratorCallable()(name)
return _ExecutorDecoratorCallable(
name=name,
config_schema=canonicalize_backcompat_args(config_schema,
'config_schema', config,
'config', '0.9.0'),
required_resource_keys=required_resource_keys,
)
def __init__(
self,
name,
input_defs,
compute_fn,
output_defs,
config=None,
description=None,
tags=None,
required_resource_keys=None,
positional_inputs=None,
metadata=None,
):
self._compute_fn = check.callable_param(compute_fn, 'compute_fn')
self._config_field = check_user_facing_opt_config_param(
config, 'config')
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
'required_resource_keys',
of_type=str))
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, 'input_defs',
InputDefinition),
output_defs=check.list_param(output_defs, 'output_defs',
OutputDefinition),
description=description,
tags=canonicalize_backcompat_args(tags, 'tags', metadata,
'metadata'),
positional_inputs=positional_inputs,
)
def get_intermediate(self,
context,
dagster_type=None,
step_output_handle=None,
runtime_type=None):
canonicalize_dagster_type = resolve_dagster_type(
canonicalize_backcompat_args(
dagster_type,
'dagster_type',
runtime_type,
'runtime_type',
)) # TODO to deprecate in 0.8.0
check.opt_inst_param(context, 'context',
SystemPipelineExecutionContext)
check.inst_param(canonicalize_dagster_type, 'dagster_type',
DagsterType)
check.inst_param(step_output_handle, 'step_output_handle',
StepOutputHandle)
check.invariant(self.has_intermediate(context, step_output_handle))
return self._intermediate_store.get_value(
context=context,
dagster_type=canonicalize_dagster_type,
paths=self._get_paths(step_output_handle),
)
def input_hydration_config(config_schema=None, required_resource_keys=None, config_cls=None):
'''Deprecated in favor of dagster_type_loader'''
rename_warning('dagster_type_loader', 'input_hydration_config', '0.10.0')
config_schema = canonicalize_backcompat_args(
config_schema, 'config_schema', config_cls, 'config_cls', '0.10.0',
)
return dagster_type_loader(config_schema, required_resource_keys)
def __init__(
self,
config,
default_value=FIELD_NO_DEFAULT_PROVIDED,
is_optional=None,
is_required=None,
description=None,
):
from .validate import validate_config
from .post_process import post_process_config
self.config_type = check.inst(self._resolve_config_arg(config), ConfigType)
self.description = check.opt_str_param(description, 'description')
check.opt_bool_param(is_optional, 'is_optional')
check.opt_bool_param(is_required, 'is_required')
canonical_is_required = canonicalize_backcompat_args(
new_val=is_required,
new_arg='is_required',
old_val=is_optional,
old_arg='is_optional',
coerce_old_to_new=lambda val: not val,
additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.',
)
if canonical_is_required is True:
check.param_invariant(
default_value == FIELD_NO_DEFAULT_PROVIDED,
'default_value',
'required arguments should not specify default values',
)
self._default_value = default_value
# check explicit default value
if self.default_provided:
# invoke through property in case it is callable
value = self.default_value
evr = validate_config(self.config_type, value)
if not evr.success:
raise DagsterInvalidConfigError(
'Invalid default_value for Field.', evr.errors, default_value,
)
if canonical_is_required is None:
# neither is_required nor is_optional were specified
canonical_is_required = not all_optional_type(self.config_type)
# on implicitly optional - set the default value
# by resolving the defaults of the type
if not canonical_is_required and self._default_value == FIELD_NO_DEFAULT_PROVIDED:
evr = post_process_config(self.config_type, None)
if not evr.success:
raise DagsterInvalidConfigError(
'Unable to resolve implicit default_value for Field.', evr.errors, None,
)
self._default_value = evr.value
self.is_optional = not canonical_is_required
def make_airflow_dag_for_recon_repo(
recon_repo,
job_name,
run_config=None,
mode=None,
dag_id=None,
dag_description=None,
dag_kwargs=None,
op_kwargs=None,
pipeline_name=None,
):
job_name = canonicalize_backcompat_args(
new_val=job_name,
new_arg="job_name",
old_val=pipeline_name,
old_arg="pipeline_name",
breaking_version="future versions",
coerce_old_to_new=lambda val: val,
)
return _make_airflow_dag(
recon_repo=recon_repo,
job_name=job_name,
run_config=run_config,
mode=mode,
dag_id=dag_id,
dag_description=dag_description,
dag_kwargs=dag_kwargs,
op_kwargs=op_kwargs,
)
def scaffold(module_name, pipeline_name, output_path, config, preset, job_name):
"""Creates a DAG file for a specified dagster pipeline"""
job_name = canonicalize_backcompat_args(
new_val=job_name,
new_arg="job_name",
old_val=pipeline_name,
old_arg="pipeline_name",
breaking_version="future versions",
coerce_old_to_new=lambda val: val,
)
check.invariant(job_name is not None, "You must specify either --job-name or --pipeline-name.")
check.tuple_param(config, "config", of_type=str)
check.invariant(isinstance(config, tuple))
check.invariant(
output_path is not None,
"You must specify --output-path or set AIRFLOW_HOME to use this script.",
)
run_config = construct_environment_yaml(preset, config, job_name, module_name)
file_contents = construct_scaffolded_file_contents(module_name, job_name, run_config)
# Ensure output_path/dags exists
dags_path = os.path.join(os.path.expanduser(output_path), "dags")
if not os.path.isdir(dags_path):
os.makedirs(dags_path)
dag_file = os.path.join(os.path.expanduser(output_path), "dags", job_name + ".py")
click.echo("Wrote DAG scaffold to file: %s" % dag_file)
with open(dag_file, "wb") as f:
f.write(file_contents)
def _wrap(resource_fn):
return _ResourceDecoratorCallable(
config_schema=canonicalize_backcompat_args(config_schema,
'config_schema', config,
'config', '0.9.0'),
description=description,
)(resource_fn)
def __new__(
cls,
name,
dagster_type=None,
source_type=None,
source_handles=None,
config_data=None,
runtime_type=None,
):
canonicalize_dagster_type = canonicalize_backcompat_args(
dagster_type, 'dagster_type', runtime_type, 'runtime_type',
)
return super(StepInput, cls).__new__(
cls,
name=check.str_param(name, 'name'),
runtime_type=check.inst_param(
canonicalize_dagster_type, 'runtime_type', DagsterType
), # TODO to deprecate in 0.8.0
dagster_type=check.inst_param(canonicalize_dagster_type, 'dagster_type', DagsterType),
source_type=check.inst_param(source_type, 'source_type', StepInputSourceType),
source_handles=check.opt_list_param(
source_handles, 'source_handles', of_type=StepOutputHandle
),
config_data=config_data, # can be any type
)
def __new__(
cls,
name,
is_persistent,
required_resource_keys,
config_schema=None,
system_storage_creation_fn=None,
config=None,
):
return super(SystemStorageDefinition, cls).__new__(
cls,
name=check.str_param(name, 'name'),
is_persistent=check.bool_param(is_persistent, 'is_persistent'),
config_schema=canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema,
'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
),
system_storage_creation_fn=check.opt_callable_param(
system_storage_creation_fn, 'system_storage_creation_fn'),
required_resource_keys=frozenset(
check.set_param(required_resource_keys,
'required_resource_keys',
of_type=str)),
)
def _wrap(logger_fn):
return LoggerDefinition(
logger_fn=logger_fn,
config_schema=canonicalize_backcompat_args(config_schema,
'config_schema', config,
'config', '0.9.0'),
description=description,
)
def define_dagstermill_solid(
name,
notebook_path,
input_defs=None,
output_defs=None,
config=None,
required_resource_keys=None,
output_notebook=None,
config_schema=None,
):
'''Wrap a Jupyter notebook in a solid.
Arguments:
name (str): The name of the solid.
notebook_path (str): Path to the backing notebook.
input_defs (Optional[List[InputDefinition]]): The solid's inputs.
output_defs (Optional[List[OutputDefinition]]): The solid's outputs. Your notebook should
call :py:func:`~dagstermill.yield_result` to yield each of these outputs.
required_resource_keys (Optional[Set[str]]): The string names of any required resources.
output_notebook (Optional[str]): If set, will be used as the name of an injected output of
type :py:class:`~dagster.FileHandle` that will point to the executed notebook (in
addition to the :py:class:`~dagster.Materialization` that is always created). This
respects the :py:class:`~dagster.core.storage.file_manager.FileManager` configured on
the pipeline system storage, so, e.g., if :py:class:`~dagster_aws.s3.s3_system_storage`
is configured, the output will be a :py:class:`~dagster_aws.s3.S3FileHandle`.
Returns:
:py:class:`~dagster.SolidDefinition`
'''
check.str_param(name, 'name')
check.str_param(notebook_path, 'notebook_path')
input_defs = check.opt_list_param(input_defs, 'input_defs', of_type=InputDefinition)
output_defs = check.opt_list_param(output_defs, 'output_defs', of_type=OutputDefinition)
required_resource_keys = check.opt_set_param(
required_resource_keys, 'required_resource_keys', of_type=str
)
return SolidDefinition(
name=name,
input_defs=input_defs,
compute_fn=_dm_solid_compute(name, notebook_path, output_notebook),
output_defs=output_defs
+ (
[OutputDefinition(dagster_type=FileHandle, name=output_notebook)]
if output_notebook
else []
),
config_schema=canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema, 'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
),
required_resource_keys=required_resource_keys,
description='This solid is backed by the notebook at {path}'.format(path=notebook_path),
tags={'notebook_path': notebook_path, 'kind': 'ipynb'},
)
def __new__(
cls,
kind,
key,
name,
description,
display_name,
is_builtin,
type_param_keys,
loader_schema_key=None,
materializer_schema_key=None,
input_hydration_schema_key=None,
output_materialization_schema_key=None,
):
return super(DagsterTypeSnap, cls).__new__(
cls,
kind=check.inst_param(kind, 'kind', DagsterTypeKind),
key=check.str_param(key, 'key'),
name=check.opt_str_param(name, 'name'),
display_name=check.str_param(display_name, 'display_name'),
description=check.opt_str_param(description, 'description'),
is_builtin=check.bool_param(is_builtin, 'is_builtin'),
type_param_keys=check.list_param(type_param_keys,
'type_param_keys',
of_type=str),
loader_schema_key=canonicalize_backcompat_args(
check.opt_str_param(loader_schema_key, 'loader_schema_key'),
'loader_schema_key',
check.opt_str_param(input_hydration_schema_key,
'input_hydration_schema_key'),
'input_hydration_schema_key',
'0.10.0',
),
materializer_schema_key=canonicalize_backcompat_args(
check.opt_str_param(materializer_schema_key,
'materializer_schema_key'),
'materializer_schema_key',
check.opt_str_param(output_materialization_schema_key,
'output_materialization_schema_key'),
'output_materialization_schema_key',
'0.10.0',
),
)
def __new__(
cls,
kind,
key,
name,
description,
display_name,
is_builtin,
type_param_keys,
loader_schema_key=None,
materializer_schema_key=None,
input_hydration_schema_key=None,
output_materialization_schema_key=None,
):
return super(DagsterTypeSnap, cls).__new__(
cls,
kind=check.inst_param(kind, "kind", DagsterTypeKind),
key=check.str_param(key, "key"),
name=check.opt_str_param(name, "name"),
display_name=check.str_param(display_name, "display_name"),
description=check.opt_str_param(description, "description"),
is_builtin=check.bool_param(is_builtin, "is_builtin"),
type_param_keys=check.list_param(type_param_keys,
"type_param_keys",
of_type=str),
loader_schema_key=canonicalize_backcompat_args(
check.opt_str_param(loader_schema_key, "loader_schema_key"),
"loader_schema_key",
check.opt_str_param(input_hydration_schema_key,
"input_hydration_schema_key"),
"input_hydration_schema_key",
"0.10.0",
),
materializer_schema_key=canonicalize_backcompat_args(
check.opt_str_param(materializer_schema_key,
"materializer_schema_key"),
"materializer_schema_key",
check.opt_str_param(output_materialization_schema_key,
"output_materialization_schema_key"),
"output_materialization_schema_key",
"0.10.0",
),
)
def _logged_pipeline_launch_command(config, preset, mode, instance, kwargs):
check.inst_param(instance, 'instance', DagsterInstance)
env = (
canonicalize_backcompat_args(
(config if config else None),
'--config',
(kwargs.get('env') if kwargs.get('env') else None),
'--env',
'0.9.0',
stacklevel=2, # this stacklevel can point the warning to this line
) or tuple() # back to default empty tuple
)
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
repo_location = get_repository_location_from_kwargs(kwargs, instance)
external_repo = get_external_repository_from_repo_location(
repo_location, kwargs.get('repository'))
external_pipeline = get_external_pipeline_from_external_repo(
external_repo,
kwargs.get('pipeline'),
)
log_external_repo_stats(
instance=instance,
external_pipeline=external_pipeline,
external_repo=external_repo,
source='pipeline_launch_command',
)
if preset:
if env:
raise click.UsageError('Can not use --preset with --config.')
preset = external_pipeline.get_preset(preset)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
solid_selection = get_solid_selection_from_args(kwargs)
pipeline_run = _create_external_pipeline_run(
instance=instance,
repo_location=repo_location,
external_repo=external_repo,
external_pipeline=external_pipeline,
run_config=get_run_config_from_env_file_list(env),
mode=mode,
preset=preset,
tags=run_tags,
solid_selection=solid_selection,
)
return instance.launch_run(pipeline_run.run_id, external_pipeline)
def is_new(old_flag=None, new_flag=None):
actual_new_flag = canonicalize_backcompat_args(
new_val=new_flag,
new_arg='new_flag',
old_val=old_flag,
old_arg='old_flag',
coerce_old_to_new=lambda val: not val,
additional_warn_txt='Will remove at next release.',
)
return actual_new_flag
def is_new(old_flag=None, new_flag=None, include_additional_warn_txt=True):
actual_new_flag = canonicalize_backcompat_args(
new_val=new_flag,
new_arg="new_flag",
old_val=old_flag,
old_arg="old_flag",
breaking_version="0.9.0",
coerce_old_to_new=lambda val: not val,
additional_warn_txt="Will remove at next release." if include_additional_warn_txt else None,
)
return actual_new_flag
def __new__(cls, config_fn, config_schema=None, config=None):
return super(ConfigMapping, cls).__new__(
cls,
config_fn=check.callable_param(config_fn, 'config_fn'),
config_schema=canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema,
'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
),
)
def __init__(self,
resource_fn,
config_schema=None,
description=None,
config=None):
self._resource_fn = check.callable_param(resource_fn, 'resource_fn')
self._config_schema = canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema, 'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
)
self._description = check.opt_str_param(description, 'description')
def make_airflow_dag_containerized_for_recon_repo(
recon_repo,
job_name,
image,
run_config=None,
mode=None,
dag_id=None,
dag_description=None,
dag_kwargs=None,
op_kwargs=None,
instance=None,
pipeline_name=None,
):
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
check.str_param(job_name, "job_name")
check.str_param(image, "image")
check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
check.opt_str_param(dag_id, "dag_id")
check.opt_str_param(dag_description, "dag_description")
check.opt_dict_param(dag_kwargs, "dag_kwargs")
op_kwargs = check.opt_dict_param(op_kwargs, "op_kwargs", key_type=str)
check.opt_str_param(pipeline_name, "pipeline_name")
op_kwargs["image"] = image
job_name = canonicalize_backcompat_args(
new_val=job_name,
new_arg="job_name",
old_val=pipeline_name,
old_arg="pipeline_name",
breaking_version="future versions",
coerce_old_to_new=lambda val: val,
)
return _make_airflow_dag(
recon_repo=recon_repo,
job_name=job_name,
run_config=run_config,
mode=mode,
dag_id=dag_id,
dag_description=dag_description,
dag_kwargs=dag_kwargs,
op_kwargs=op_kwargs,
operator=DagsterDockerOperator,
instance=instance,
)
def __new__(
cls,
name,
pipeline_name,
partition_fn,
solid_selection=None,
mode=None,
run_config_fn_for_partition=None,
tags_fn_for_partition=lambda _partition: {},
environment_dict_fn_for_partition=None,
):
def _wrap(x):
if isinstance(x, Partition):
return x
if isinstance(x, str):
return Partition(x)
raise DagsterInvalidDefinitionError(
'Expected <Partition> | <str>, received {type}'.format(type=type(x))
)
run_config_fn_for_partition = canonicalize_backcompat_args(
run_config_fn_for_partition,
'run_config_fn_for_partition',
environment_dict_fn_for_partition,
'environment_dict_fn_for_partition',
'0.9.0',
) or (lambda _partition: {})
return super(PartitionSetDefinition, cls).__new__(
cls,
name=check.str_param(name, 'name'),
pipeline_name=check.str_param(pipeline_name, 'pipeline_name'),
partition_fn=lambda: [
_wrap(x) for x in check.callable_param(partition_fn, 'partition_fn')()
],
solid_selection=check.opt_nullable_list_param(
solid_selection, 'solid_selection', of_type=str
),
mode=check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME),
user_defined_run_config_fn_for_partition=check.callable_param(
run_config_fn_for_partition, 'run_config_fn_for_partition'
),
user_defined_tags_fn_for_partition=check.callable_param(
tags_fn_for_partition, 'tags_fn_for_partition'
),
)
def __init__(
self, dagster_type=None, name=None, description=None, is_optional=None, is_required=None
):
self._name = check_valid_name(check.opt_str_param(name, 'name', DEFAULT_OUTPUT))
self._runtime_type = resolve_dagster_type(dagster_type)
self._description = check.opt_str_param(description, 'description')
check.opt_bool_param(is_optional, 'is_optional')
check.opt_bool_param(is_required, 'is_required')
canonical_is_required = canonicalize_backcompat_args(
new_val=is_required,
new_arg='is_required',
old_val=is_optional,
old_arg='is_optional',
coerce_old_to_new=lambda val: not val,
additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.',
)
self._optional = False if (canonical_is_required is None) else not canonical_is_required
def get_intermediate(self,
context,
dagster_type=None,
step_output_handle=None,
runtime_type=None):
canonicalize_dagster_type = canonicalize_backcompat_args(
dagster_type,
'dagster_type',
runtime_type,
'runtime_type',
) # TODO to deprecate in 0.8.0
check.opt_inst_param(context, 'context',
SystemPipelineExecutionContext)
check.opt_inst_param(canonicalize_dagster_type, 'dagster_type',
DagsterType)
check.inst_param(step_output_handle, 'step_output_handle',
StepOutputHandle)
return self.values[step_output_handle]
def from_files(name,
environment_files=None,
config_files=None,
solid_selection=None,
mode=None,
tags=None):
'''Static constructor for presets from YAML files.
Args:
name (str): The name of this preset. Must be unique in the presets defined on a given
pipeline.
config_files (Optional[List[str]]): List of paths or glob patterns for yaml files
to load and parse as the environment config for this preset.
solid_selection (Optional[List[str]]): A list of solid subselection (including single
solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']``
mode (Optional[str]): The mode to apply when executing this preset. (default:
'default')
tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset.
Returns:
PresetDefinition: A PresetDefinition constructed from the provided YAML files.
Raises:
DagsterInvariantViolationError: When one of the YAML files is invalid and has a parse
error.
'''
check.str_param(name, 'name')
config_files = canonicalize_backcompat_args(config_files,
'config_files',
environment_files,
'environment_files',
'0.9.0')
config_files = check.opt_list_param(config_files, 'config_files')
solid_selection = check.opt_nullable_list_param(solid_selection,
'solid_selection',
of_type=str)
mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME)
merged = config_from_files(config_files)
return PresetDefinition(name, merged, solid_selection, mode, tags)
