def __init__(
self,
server_termination_event,
loadable_target_origin=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
fixed_server_id=None,
):
super(DagsterApiServer, self).__init__()
check.bool_param(heartbeat, "heartbeat")
check.int_param(heartbeat_timeout, "heartbeat_timeout")
check.invariant(heartbeat_timeout > 0,
"heartbeat_timeout must be greater than 0")
self._server_termination_event = check.inst_param(
server_termination_event, "server_termination_event",
seven.ThreadingEventType)
self._loadable_target_origin = check.opt_inst_param(
loadable_target_origin, "loadable_target_origin",
LoadableTargetOrigin)
# Each server is initialized with a unique UUID. This UUID is used by clients to track when
# servers are replaced and is used for cache invalidation and reloading.
self._server_id = check.opt_str_param(fixed_server_id,
"fixed_server_id",
str(uuid.uuid4()))
# Client tells the server to shutdown by calling ShutdownServer (or by failing to send a
# hearbeat, at which point this event is set. The cleanup thread will then set the server
# termination event once all current executions have finished, which will stop the server)
self._shutdown_once_executions_finish_event = threading.Event()
# Dict[str, (multiprocessing.Process, DagsterInstance)]
self._executions = {}
# Dict[str, multiprocessing.Event]
self._termination_events = {}
self._termination_times = {}
self._execution_lock = threading.Lock()
self._repository_symbols_and_code_pointers = LazyRepositorySymbolsAndCodePointers(
loadable_target_origin)
if not lazy_load_user_code:
self._repository_symbols_and_code_pointers.load()
self.__last_heartbeat_time = time.time()
if heartbeat:
self.__heartbeat_thread = threading.Thread(
target=self._heartbeat_thread,
args=(heartbeat_timeout, ),
)
self.__heartbeat_thread.daemon = True
self.__heartbeat_thread.start()
else:
self.__heartbeat_thread = None
self.__cleanup_thread = threading.Thread(
target=self._cleanup_thread,
args=(),
)
self.__cleanup_thread.daemon = True
self.__cleanup_thread.start()
def __init__(self, instance, scheduled_execution_time):
super(ScheduleExecutionContext, self).__init__(
check.inst_param(instance, "instance", DagsterInstance))
self._scheduled_execution_time = check.opt_inst_param(
scheduled_execution_time, "scheduled_execution_time", datetime)
def __init__(
self,
type_check_fn,
key=None,
name=None,
is_builtin=False,
description=None,
loader=None,
materializer=None,
serialization_strategy=None,
auto_plugins=None,
required_resource_keys=None,
kind=DagsterTypeKind.REGULAR,
# Graveyard is below
input_hydration_config=None,
output_materialization_config=None,
):
check.opt_str_param(key, "key")
check.opt_str_param(name, "name")
check.invariant(not (name is None and key is None), "Must set key or name")
if name is None:
check.param_invariant(
bool(key), "key", "If name is not provided, must provide key.",
)
self.key, self.name = key, None
elif key is None:
check.param_invariant(
bool(name), "name", "If key is not provided, must provide name.",
)
self.key, self.name = name, name
else:
check.invariant(key and name)
self.key, self.name = key, name
self.description = check.opt_str_param(description, "description")
self.loader = canonicalize_backcompat_args(
check.opt_inst_param(loader, "loader", DagsterTypeLoader),
"loader",
check.opt_inst_param(
input_hydration_config, "input_hydration_config", DagsterTypeLoader
),
"input_hydration_config",
"0.10.0",
)
self.materializer = canonicalize_backcompat_args(
check.opt_inst_param(materializer, "materializer", DagsterTypeMaterializer),
"materializer",
check.opt_inst_param(
output_materialization_config,
"output_materialization_config",
DagsterTypeMaterializer,
),
"output_materialization_config",
"0.10.0",
)
self.serialization_strategy = check.opt_inst_param(
serialization_strategy,
"serialization_strategy",
SerializationStrategy,
PickleSerializationStrategy(),
)
self.required_resource_keys = check.opt_set_param(
required_resource_keys, "required_resource_keys",
)
self._type_check_fn = check.callable_param(type_check_fn, "type_check_fn")
_validate_type_check_fn(self._type_check_fn, self.name)
auto_plugins = check.opt_list_param(auto_plugins, "auto_plugins", of_type=type)
check.param_invariant(
all(
issubclass(auto_plugin_type, TypeStoragePlugin) for auto_plugin_type in auto_plugins
),
"auto_plugins",
)
self.auto_plugins = auto_plugins
self.is_builtin = check.bool_param(is_builtin, "is_builtin")
check.invariant(
self.display_name is not None,
"All types must have a valid display name, got None for key {}".format(key),
)
self.kind = check.inst_param(kind, "kind", DagsterTypeKind)
def from_dagster_event_record(graphene_info, event_record, dauphin_pipeline,
execution_plan):
# Lots of event types. Pylint thinks there are too many branches
# pylint: disable=too-many-branches
check.inst_param(event_record, 'event_record', EventRecord)
check.param_invariant(event_record.is_dagster_event, 'event_record')
check.opt_inst_param(dauphin_pipeline, 'dauphin_pipeline',
graphene_info.schema.type_named('Pipeline'))
check.opt_inst_param(execution_plan, 'execution_plan', ExecutionPlan)
dagster_event = event_record.dagster_event
basic_params = construct_basic_params(graphene_info, event_record,
execution_plan)
if dagster_event.event_type == DagsterEventType.STEP_START:
return graphene_info.schema.type_named('ExecutionStepStartEvent')(
**basic_params)
elif dagster_event.event_type == DagsterEventType.STEP_SKIPPED:
return graphene_info.schema.type_named('ExecutionStepSkippedEvent')(
**basic_params)
elif dagster_event.event_type == DagsterEventType.STEP_SUCCESS:
return graphene_info.schema.type_named('ExecutionStepSuccessEvent')(
**basic_params)
elif dagster_event.event_type == DagsterEventType.STEP_INPUT:
input_data = dagster_event.event_specific_data
return graphene_info.schema.type_named('ExecutionStepInputEvent')(
input_name=input_data.input_name,
type_check=input_data.type_check_data,
**basic_params)
elif dagster_event.event_type == DagsterEventType.STEP_OUTPUT:
output_data = dagster_event.step_output_data
return graphene_info.schema.type_named('ExecutionStepOutputEvent')(
output_name=output_data.output_name,
type_check=output_data.type_check_data,
# parens make black not put trailing commas, which in turn break py27
# fmt: off
**(basic_params)
# fmt: on
)
elif dagster_event.event_type == DagsterEventType.STEP_MATERIALIZATION:
materialization = dagster_event.step_materialization_data.materialization
return graphene_info.schema.type_named('StepMaterializationEvent')(
materialization=materialization, **basic_params)
elif dagster_event.event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
expectation_result = dagster_event.event_specific_data.expectation_result
return graphene_info.schema.type_named('StepExpectationResultEvent')(
expectation_result=expectation_result, **(basic_params))
elif dagster_event.event_type == DagsterEventType.STEP_FAILURE:
check.inst(dagster_event.step_failure_data, StepFailureData)
return graphene_info.schema.type_named('ExecutionStepFailureEvent')(
error=graphene_info.schema.type_named('PythonError')(
dagster_event.step_failure_data.error),
# parens make black not put trailing commas, which in turn break py27
# fmt: off
**(basic_params)
# fmt: on
)
elif dagster_event.event_type == DagsterEventType.PIPELINE_START:
return graphene_info.schema.type_named('PipelineStartEvent')(
pipeline=dauphin_pipeline, **basic_params)
elif dagster_event.event_type == DagsterEventType.PIPELINE_SUCCESS:
return graphene_info.schema.type_named('PipelineSuccessEvent')(
pipeline=dauphin_pipeline, **basic_params)
elif dagster_event.event_type == DagsterEventType.PIPELINE_FAILURE:
return graphene_info.schema.type_named('PipelineFailureEvent')(
pipeline=dauphin_pipeline, **basic_params)
elif dagster_event.event_type == DagsterEventType.PIPELINE_PROCESS_START:
process_data = dagster_event.pipeline_process_start_data
return graphene_info.schema.type_named('PipelineProcessStartEvent')(
pipeline=dauphin_pipeline,
pipeline_name=process_data.pipeline_name,
run_id=process_data.run_id,
**basic_params)
elif dagster_event.event_type == DagsterEventType.PIPELINE_PROCESS_STARTED:
process_data = dagster_event.pipeline_process_started_data
return graphene_info.schema.type_named('PipelineProcessStartedEvent')(
pipeline=dauphin_pipeline,
process_id=process_data.process_id,
**basic_params)
elif dagster_event.event_type == DagsterEventType.PIPELINE_PROCESS_EXITED:
process_data = dagster_event.pipeline_process_exited_data
return graphene_info.schema.type_named('PipelineProcessExitedEvent')(
pipeline=dauphin_pipeline,
process_id=process_data.process_id,
**basic_params)
elif dagster_event.event_type == DagsterEventType.PIPELINE_INIT_FAILURE:
return graphene_info.schema.type_named('PipelineInitFailureEvent')(
pipeline=dauphin_pipeline,
error=graphene_info.schema.type_named('PythonError')(
dagster_event.pipeline_init_failure_data.error),
# parens make black not put trailing commas, which in turn break py27
# fmt: off
**(basic_params)
# fmt: on
)
elif dagster_event.event_type == DagsterEventType.OBJECT_STORE_OPERATION:
operation_result = dagster_event.event_specific_data
return graphene_info.schema.type_named('ObjectStoreOperationEvent')(
operation_result=operation_result, **basic_params)
elif dagster_event.event_type == DagsterEventType.ENGINE_EVENT:
return graphene_info.schema.type_named('EngineEvent')(
metadataEntries=_to_dauphin_metadata_entries(
dagster_event.event_specific_data.metadata_entries),
**basic_params)
else:
raise Exception(
'Unknown DAGSTER_EVENT type {inner_type} found in logs'.format(
inner_type=dagster_event.event_type))
def create_execution_structure(solid_defs, dependencies_dict,
container_definition):
'''This builder takes the dependencies dictionary specified during creation of the
PipelineDefinition object and builds (1) the execution structure and (2) a solid dependency
dictionary.
For example, for the following dependencies:
dep_dict = {
SolidInvocation('giver'): {},
SolidInvocation('sleeper', alias='sleeper_1'): {
'units': DependencyDefinition('giver', 'out_1')
},
SolidInvocation('sleeper', alias='sleeper_2'): {
'units': DependencyDefinition('giver', 'out_2')
},
SolidInvocation('sleeper', alias='sleeper_3'): {
'units': DependencyDefinition('giver', 'out_3')
},
SolidInvocation('sleeper', alias='sleeper_4'): {
'units': DependencyDefinition('giver', 'out_4')
},
SolidInvocation('total'): {
'in_1': DependencyDefinition('sleeper_1', 'total'),
'in_2': DependencyDefinition('sleeper_2', 'total'),
'in_3': DependencyDefinition('sleeper_3', 'total'),
'in_4': DependencyDefinition('sleeper_4', 'total'),
},
},
This will create:
pipeline_solid_dict = {
'giver': <dagster.core.definitions.dependency.Solid object>,
'sleeper_1': <dagster.core.definitions.dependency.Solid object>,
'sleeper_2': <dagster.core.definitions.dependency.Solid object>,
'sleeper_3': <dagster.core.definitions.dependency.Solid object>,
'sleeper_4': <dagster.core.definitions.dependency.Solid object>,
'total': <dagster.core.definitions.dependency.Solid object>
}
as well as a dagster.core.definitions.dependency.DependencyStructure object.
'''
from .solid import ISolidDefinition, CompositeSolidDefinition
check.list_param(solid_defs, 'solid_defs', of_type=ISolidDefinition)
check.dict_param(
dependencies_dict,
'dependencies_dict',
key_type=six.string_types + (SolidInvocation, ),
value_type=dict,
)
# container_definition is none in the context of a pipeline
check.opt_inst_param(container_definition, 'container_definition',
CompositeSolidDefinition)
# Same as dep_dict but with SolidInvocation replaced by alias string
aliased_dependencies_dict = {}
# Keep track of solid name -> all aliases used and alias -> name
name_to_aliases = defaultdict(set)
alias_to_solid_instance = {}
alias_to_name = {}
for solid_key, input_dep_dict in dependencies_dict.items():
# We allow deps of the form dependencies={'foo': DependencyDefition('bar')}
# Here, we replace 'foo' with SolidInvocation('foo')
if not isinstance(solid_key, SolidInvocation):
solid_key = SolidInvocation(solid_key)
alias = solid_key.alias or solid_key.name
name_to_aliases[solid_key.name].add(alias)
alias_to_solid_instance[alias] = solid_key
alias_to_name[alias] = solid_key.name
aliased_dependencies_dict[alias] = input_dep_dict
for dependency in input_dep_dict.values():
for dep in dependency.get_definitions():
name_to_aliases[dep.solid].add(dep.solid)
pipeline_solid_dict = _build_pipeline_solid_dict(solid_defs,
name_to_aliases,
alias_to_solid_instance,
container_definition)
_validate_dependencies(aliased_dependencies_dict, pipeline_solid_dict,
alias_to_name)
dependency_structure = DependencyStructure.from_definitions(
pipeline_solid_dict, aliased_dependencies_dict)
return dependency_structure, pipeline_solid_dict
def construct_dagster_k8s_job(
job_config,
args,
job_name,
user_defined_k8s_config=None,
pod_name=None,
component=None,
env_vars=None,
):
"""Constructs a Kubernetes Job object for a dagster-graphql invocation.
Args:
job_config (DagsterK8sJobConfig): Job configuration to use for constructing the Kubernetes
Job object.
args (List[str]): CLI arguments to use with dagster-graphql in this Job.
job_name (str): The name of the Job. Note that this name must be <= 63 characters in length.
resources (Dict[str, Dict[str, str]]): The resource requirements for the container
pod_name (str, optional): The name of the Pod. Note that this name must be <= 63 characters
in length. Defaults to "<job_name>-pod".
component (str, optional): The name of the component, used to provide the Job label
app.kubernetes.io/component. Defaults to None.
env_vars(Dict[str, str]): Additional environment variables to add to the K8s Container.
Returns:
kubernetes.client.V1Job: A Kubernetes Job object.
"""
check.inst_param(job_config, "job_config", DagsterK8sJobConfig)
check.list_param(args, "args", of_type=str)
check.str_param(job_name, "job_name")
user_defined_k8s_config = check.opt_inst_param(
user_defined_k8s_config,
"user_defined_k8s_config",
UserDefinedDagsterK8sConfig,
UserDefinedDagsterK8sConfig(),
)
pod_name = check.opt_str_param(pod_name,
"pod_name",
default=job_name + "-pod")
check.opt_str_param(component, "component")
check.opt_dict_param(env_vars, "env_vars", key_type=str, value_type=str)
check.invariant(
len(job_name) <= MAX_K8S_NAME_LEN,
"job_name is %d in length; Kubernetes Jobs cannot be longer than %d characters."
% (len(job_name), MAX_K8S_NAME_LEN),
)
check.invariant(
len(pod_name) <= MAX_K8S_NAME_LEN,
"job_name is %d in length; Kubernetes Pods cannot be longer than %d characters."
% (len(pod_name), MAX_K8S_NAME_LEN),
)
# See: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/
dagster_labels = {
"app.kubernetes.io/name": "dagster",
"app.kubernetes.io/instance": "dagster",
"app.kubernetes.io/version": dagster_version,
"app.kubernetes.io/part-of": "dagster",
}
if component:
dagster_labels["app.kubernetes.io/component"] = component
env = [
kubernetes.client.V1EnvVar(name="DAGSTER_HOME",
value=job_config.dagster_home)
]
if job_config.postgres_password_secret:
env.append(
kubernetes.client.V1EnvVar(
name=DAGSTER_PG_PASSWORD_ENV_VAR,
value_from=kubernetes.client.V1EnvVarSource(
secret_key_ref=kubernetes.client.V1SecretKeySelector(
name=job_config.postgres_password_secret,
key=DAGSTER_PG_PASSWORD_SECRET_KEY)),
))
additional_k8s_env_vars = []
if env_vars:
for key, value in env_vars.items():
additional_k8s_env_vars.append(
kubernetes.client.V1EnvVar(name=key, value=value))
job_container = kubernetes.client.V1Container(
name=job_name,
image=job_config.job_image,
args=args,
image_pull_policy=job_config.image_pull_policy,
env=env + additional_k8s_env_vars,
env_from=job_config.env_from_sources,
volume_mounts=[
kubernetes.client.V1VolumeMount(
name="dagster-instance",
mount_path="{dagster_home}/dagster.yaml".format(
dagster_home=job_config.dagster_home),
sub_path="dagster.yaml",
)
] + [
kubernetes.client.V1VolumeMount(
name=mount["name"],
mount_path=mount["path"],
sub_path=mount["sub_path"],
) for mount in job_config.volume_mounts
],
**user_defined_k8s_config.container_config,
)
volumes = [
kubernetes.client.V1Volume(
name="dagster-instance",
config_map=kubernetes.client.V1ConfigMapVolumeSource(
name=job_config.instance_config_map),
)
] + [
kubernetes.client.V1Volume(
name=mount["name"],
config_map=kubernetes.client.V1ConfigMapVolumeSource(
name=mount["configmap"]),
) if mount.get("configmap") else kubernetes.client.V1Volume(
name=mount["name"],
secret=kubernetes.client.V1SecretVolumeSource(
secret_name=mount["secret"]),
) for mount in job_config.volume_mounts
]
# If the user has defined custom labels, remove them from the pod_template_spec_metadata
# key and merge them with the dagster labels
user_defined_pod_template_labels = user_defined_k8s_config.pod_template_spec_metadata.pop(
"labels", {})
template = kubernetes.client.V1PodTemplateSpec(
metadata=kubernetes.client.V1ObjectMeta(
name=pod_name,
labels=merge_dicts(dagster_labels,
user_defined_pod_template_labels),
**user_defined_k8s_config.pod_template_spec_metadata,
),
spec=kubernetes.client.V1PodSpec(
image_pull_secrets=[
kubernetes.client.V1LocalObjectReference(name=x["name"])
for x in job_config.image_pull_secrets
],
service_account_name=job_config.service_account_name,
restart_policy="Never",
containers=[job_container],
volumes=volumes,
**user_defined_k8s_config.pod_spec_config,
),
)
job = kubernetes.client.V1Job(
api_version="batch/v1",
kind="Job",
metadata=kubernetes.client.V1ObjectMeta(
name=job_name,
labels=dagster_labels,
**user_defined_k8s_config.job_metadata),
spec=kubernetes.client.V1JobSpec(
template=template,
backoff_limit=K8S_JOB_BACKOFF_LIMIT,
ttl_seconds_after_finished=K8S_JOB_TTL_SECONDS_AFTER_FINISHED,
**user_defined_k8s_config.job_spec_config,
),
**user_defined_k8s_config.job_config,
)
return job
def add_pending_invocation(self, solid):
solid = check.opt_inst_param(solid, "solid", CallableSolidNode)
solid_name = solid.given_alias if solid.given_alias else solid.solid_def.name
self._pending_invocations[solid_name] = solid
def get_logging_tags(run_config, pipeline):
check.opt_inst_param(run_config, 'run_config', RunConfig)
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
return merge_dicts({'pipeline': pipeline.name},
run_config.tags if run_config else {})
def hourly_schedule(
pipeline_name: str,
start_date: datetime.datetime,
name: Optional[str] = None,
execution_time: datetime.time = datetime.time(0, 0),
tags_fn_for_date: Optional[Callable[[datetime.datetime], Optional[Dict[str, str]]]] = None,
solid_selection: Optional[List[str]] = None,
mode: Optional[str] = "default",
should_execute: Optional[Callable[["ScheduleExecutionContext"], bool]] = None,
environment_vars: Optional[Dict[str, str]] = None,
end_date: Optional[str] = None,
execution_timezone: Optional[str] = None,
partition_hours_offset: Optional[int] = 1,
description: Optional[str] = None,
) -> Callable[[Callable[[datetime.datetime], Dict[str, Any]]], PartitionScheduleDefinition]:
"""Create a partitioned schedule that runs hourly.
The decorated function should accept a datetime object as its only argument. The datetime
represents the date partition that it's meant to run on.
The decorated function should return a run configuration dictionary, which will be used as
configuration for the scheduled run.
The decorator produces a :py:class:`~dagster.PartitionScheduleDefinition`.
Args:
pipeline_name (str): The name of the pipeline to execute when the schedule runs.
start_date (datetime.datetime): The date from which to run the schedule.
name (Optional[str]): The name of the schedule to create. By default, this will be the name
of the decorated function.
execution_time (datetime.time): The time at which to execute the schedule. Only the minutes
component will be respected -- the hour should be 0, and will be ignored if it is not 0.
tags_fn_for_date (Optional[Callable[[datetime.datetime], Optional[Dict[str, str]]]]): A
function that generates tags to attach to the schedules runs. Takes the date of the
schedule run and returns a dictionary of tags (string key-value pairs).
solid_selection (Optional[List[str]]): A list of solid subselection (including single
solid names) to execute when the schedule runs. e.g. ``['*some_solid+', 'other_solid']``
mode (Optional[str]): The pipeline mode in which to execute this schedule.
(Default: 'default')
should_execute (Optional[Callable[ScheduleExecutionContext, bool]]): A function that runs at
schedule execution tie to determine whether a schedule should execute or skip. Takes a
:py:class:`~dagster.ScheduleExecutionContext` and returns a boolean (``True`` if the
schedule should execute). Defaults to a function that always returns ``True``.
environment_vars (Optional[Dict[str, str]]): Any environment variables to set when executing
the schedule.
end_date (Optional[datetime.datetime]): The last time to run the schedule to, defaults to
current time.
execution_timezone (Optional[str]): Timezone in which the schedule should run. Only works
with DagsterDaemonScheduler, and must be set when using that scheduler.
partition_hours_offset (Optional[int]): How many hours back to go when choosing the partition
for a given schedule execution. For example, when partition_hours_offset=1, the schedule
that executes during hour N will fill in the partition for hour N-1.
(Default: 1)
description (Optional[str]): A human-readable description of the schedule.
"""
check.opt_str_param(name, "name")
check.inst_param(start_date, "start_date", datetime.datetime)
check.opt_inst_param(end_date, "end_date", datetime.datetime)
check.opt_callable_param(tags_fn_for_date, "tags_fn_for_date")
check.opt_nullable_list_param(solid_selection, "solid_selection", of_type=str)
mode = check.opt_str_param(mode, "mode", DEFAULT_MODE_NAME)
check.opt_callable_param(should_execute, "should_execute")
check.opt_dict_param(environment_vars, "environment_vars", key_type=str, value_type=str)
check.str_param(pipeline_name, "pipeline_name")
check.inst_param(execution_time, "execution_time", datetime.time)
check.opt_str_param(execution_timezone, "execution_timezone")
check.opt_int_param(partition_hours_offset, "partition_hours_offset")
check.opt_str_param(description, "description")
if start_date.minute != 0 or start_date.second != 0:
warnings.warn(
"`start_date` must be at the beginning of the hour for an hourly schedule. "
"Use `execution_time` to execute the schedule at a specific time within the hour. For "
"example, to run the schedule each hour at 15 minutes past the hour starting at 3AM "
"on 10/20/2020, your schedule definition would look like:"
"""
@hourly_schedule(
start_date=datetime.datetime(2020, 10, 20, 3),
execution_time=datetime.time(0, 15)
):
def my_schedule_definition(_):
...
"""
)
if execution_time.hour != 0:
warnings.warn(
"Hourly schedule {schedule_name} created with:\n"
"\tschedule_time=datetime.time(hour={hour}, minute={minute}, ...)."
"Since this is an hourly schedule, the hour parameter will be ignored and the schedule "
"will run on the {minute} mark for the previous hour interval. Replace "
"datetime.time(hour={hour}, minute={minute}, ...) with "
"datetime.time(minute={minute}, ...) to fix this warning."
)
cron_schedule = "{minute} * * * *".format(minute=execution_time.minute)
fmt = (
DEFAULT_HOURLY_FORMAT_WITH_TIMEZONE
if execution_timezone
else DEFAULT_HOURLY_FORMAT_WITHOUT_TIMEZONE
)
execution_time_to_partition_fn = lambda d: pendulum.instance(d).subtract(
hours=partition_hours_offset, minutes=(execution_time.minute - start_date.minute) % 60
)
partition_fn = schedule_partition_range(
start_date,
end=end_date,
cron_schedule=cron_schedule,
fmt=fmt,
timezone=execution_timezone,
execution_time_to_partition_fn=execution_time_to_partition_fn,
inclusive=(partition_hours_offset == 0),
)
def inner(fn: Callable[[datetime.datetime], Dict[str, Any]]) -> PartitionScheduleDefinition:
check.callable_param(fn, "fn")
schedule_name = name or fn.__name__
tags_fn_for_partition_value: Callable[
["Partition"], Optional[Dict[str, str]]
] = lambda partition: {}
if tags_fn_for_date:
tags_fn = cast(
Callable[[datetime.datetime], Optional[Dict[str, str]]], tags_fn_for_date
)
tags_fn_for_partition_value = lambda partition: tags_fn(partition.value)
partition_set = PartitionSetDefinition(
name="{}_partitions".format(schedule_name),
pipeline_name=pipeline_name,
partition_fn=partition_fn,
run_config_fn_for_partition=lambda partition: fn(partition.value),
solid_selection=solid_selection,
tags_fn_for_partition=tags_fn_for_partition_value,
mode=mode,
)
return partition_set.create_schedule_definition(
schedule_name,
cron_schedule,
should_execute=should_execute,
environment_vars=environment_vars,
partition_selector=create_offset_partition_selector(
execution_time_to_partition_fn=execution_time_to_partition_fn,
),
execution_timezone=execution_timezone,
description=description,
)
return inner
def monthly_schedule(
pipeline_name,
start_date,
name=None,
execution_day_of_month=1,
execution_time=datetime.time(0, 0),
tags_fn_for_date=None,
solid_selection=None,
mode="default",
should_execute=None,
environment_vars=None,
end_date=None,
):
'''Create a schedule that runs monthly.
The decorated function will be called as the ``run_config_fn`` of the underlying
:py:class:`~dagster.ScheduleDefinition` and should take a
:py:class:`~dagster.ScheduleExecutionContext` as its only argument, returning the environment
dict for the scheduled execution.
Args:
pipeline_name (str): The name of the pipeline to execute when the schedule runs.
start_date (datetime.datetime): The date from which to run the schedule.
name (Optional[str]): The name of the schedule to create.
execution_day_of_month (int): The day of the month on which to run the schedule (must be
between 0 and 31).
execution_time (datetime.time): The time at which to execute the schedule.
tags_fn_for_date (Optional[Callable[[datetime.datetime], Optional[Dict[str, str]]]]): A
function that generates tags to attach to the schedules runs. Takes the date of the
schedule run and returns a dictionary of tags (string key-value pairs).
solid_selection (Optional[List[str]]): A list of solid subselection (including single
solid names) to execute when the schedule runs. e.g. ``['*some_solid+', 'other_solid']``
mode (Optional[str]): The pipeline mode in which to execute this schedule.
(Default: 'default')
should_execute (Optional[Callable[ScheduleExecutionContext, bool]]): A function that runs at
schedule execution tie to determine whether a schedule should execute or skip. Takes a
:py:class:`~dagster.ScheduleExecutionContext` and returns a boolean (``True`` if the
schedule should execute). Defaults to a function that always returns ``True``.
environment_vars (Optional[Dict[str, str]]): Any environment variables to set when executing
the schedule.
end_date (Optional[datetime.datetime]): The last time to run the schedule to, defaults to
current time.
'''
check.opt_str_param(name, 'name')
check.inst_param(start_date, 'start_date', datetime.datetime)
check.opt_inst_param(end_date, 'end_date', datetime.datetime)
check.opt_callable_param(tags_fn_for_date, 'tags_fn_for_date')
check.opt_nullable_list_param(solid_selection,
'solid_selection',
of_type=str)
mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME)
check.opt_callable_param(should_execute, 'should_execute')
check.opt_dict_param(environment_vars,
'environment_vars',
key_type=str,
value_type=str)
check.str_param(pipeline_name, 'pipeline_name')
check.int_param(execution_day_of_month, 'execution_day')
check.inst_param(execution_time, 'execution_time', datetime.time)
if execution_day_of_month <= 0 or execution_day_of_month > 31:
raise DagsterInvalidDefinitionError(
'`execution_day_of_month={}` is not valid for monthly schedule. Execution day must be '
'between 1 and 31'.format(execution_day_of_month))
cron_schedule = '{minute} {hour} {day} * *'.format(
minute=execution_time.minute,
hour=execution_time.hour,
day=execution_day_of_month)
partition_fn = date_partition_range(start_date,
end=end_date,
delta=relativedelta(months=1),
fmt="%Y-%m")
def inner(fn):
check.callable_param(fn, 'fn')
schedule_name = name or fn.__name__
tags_fn_for_partition_value = lambda partition: {}
if tags_fn_for_date:
tags_fn_for_partition_value = lambda partition: tags_fn_for_date(
partition.value)
partition_set = PartitionSetDefinition(
name='{}_partitions'.format(schedule_name),
pipeline_name=pipeline_name,
partition_fn=partition_fn,
run_config_fn_for_partition=lambda partition: fn(partition.value),
solid_selection=solid_selection,
tags_fn_for_partition=tags_fn_for_partition_value,
mode=mode,
)
return partition_set.create_schedule_definition(
schedule_name,
cron_schedule,
should_execute=should_execute,
environment_vars=environment_vars,
)
return inner
def scoped_pipeline_context(
pipeline_def,
environment_dict,
run_config,
instance,
system_storage_data=None,
scoped_resources_builder_cm=create_resource_builder,
):
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
check.dict_param(environment_dict, 'environment_dict', key_type=str)
check.inst_param(run_config, 'run_config', RunConfig)
check.inst_param(instance, 'instance', DagsterInstance)
check.opt_inst_param(system_storage_data, 'system_storage_data',
SystemStorageData)
context_creation_data = create_context_creation_data(
pipeline_def, environment_dict, run_config, instance)
executor_config = create_executor_config(context_creation_data)
# After this try block, a Dagster exception thrown will result in a pipeline init failure event.
try:
executor_config.check_requirements(
instance, context_creation_data.system_storage_def)
log_manager = create_log_manager(context_creation_data)
with scoped_resources_builder_cm(
context_creation_data.pipeline_def,
context_creation_data.environment_config,
context_creation_data.run_config,
log_manager,
) as scoped_resources_builder:
system_storage_data = create_system_storage_data(
context_creation_data, system_storage_data,
scoped_resources_builder)
yield construct_pipeline_execution_context(
context_creation_data=context_creation_data,
scoped_resources_builder=scoped_resources_builder,
system_storage_data=system_storage_data,
log_manager=log_manager,
executor_config=executor_config,
)
except DagsterError as dagster_error:
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info
if dagster_error.is_user_code_error else sys.exc_info())
if executor_config.raise_on_error:
raise dagster_error
error_info = serializable_error_info_from_exc_info(
user_facing_exc_info)
yield DagsterEvent.pipeline_init_failure(
pipeline_name=pipeline_def.name,
failure_data=PipelineInitFailureData(error=error_info),
log_manager=_create_context_free_log_manager(
instance, run_config, pipeline_def),
)
def hourly_schedule(
pipeline_name,
start_date,
name=None,
execution_time=datetime.time(0, 0),
tags_fn_for_date=None,
solid_selection=None,
mode="default",
should_execute=None,
environment_vars=None,
end_date=None,
):
'''Create a schedule that runs hourly.
The decorated function will be called as the ``run_config_fn`` of the underlying
:py:class:`~dagster.ScheduleDefinition` and should take a
:py:class:`~dagster.ScheduleExecutionContext` as its only argument, returning the environment
dict for the scheduled execution.
Args:
pipeline_name (str): The name of the pipeline to execute when the schedule runs.
start_date (datetime.datetime): The date from which to run the schedule.
name (Optional[str]): The name of the schedule to create. By default, this will be the name
of the decorated function.
execution_time (datetime.time): The time at which to execute the schedule. Only the minutes
component will be respected -- the hour should be 0, and will be ignored if it is not 0.
tags_fn_for_date (Optional[Callable[[datetime.datetime], Optional[Dict[str, str]]]]): A
function that generates tags to attach to the schedules runs. Takes the date of the
schedule run and returns a dictionary of tags (string key-value pairs).
solid_selection (Optional[List[str]]): A list of solid subselection (including single
solid names) to execute when the schedule runs. e.g. ``['*some_solid+', 'other_solid']``
mode (Optional[str]): The pipeline mode in which to execute this schedule.
(Default: 'default')
should_execute (Optional[Callable[ScheduleExecutionContext, bool]]): A function that runs at
schedule execution tie to determine whether a schedule should execute or skip. Takes a
:py:class:`~dagster.ScheduleExecutionContext` and returns a boolean (``True`` if the
schedule should execute). Defaults to a function that always returns ``True``.
environment_vars (Optional[Dict[str, str]]): Any environment variables to set when executing
the schedule.
end_date (Optional[datetime.datetime]): The last time to run the schedule to, defaults to
current time.
'''
check.opt_str_param(name, 'name')
check.inst_param(start_date, 'start_date', datetime.datetime)
check.opt_inst_param(end_date, 'end_date', datetime.datetime)
check.opt_callable_param(tags_fn_for_date, 'tags_fn_for_date')
check.opt_nullable_list_param(solid_selection,
'solid_selection',
of_type=str)
mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME)
check.opt_callable_param(should_execute, 'should_execute')
check.opt_dict_param(environment_vars,
'environment_vars',
key_type=str,
value_type=str)
check.str_param(pipeline_name, 'pipeline_name')
check.inst_param(execution_time, 'execution_time', datetime.time)
if execution_time.hour != 0:
warnings.warn(
"Hourly schedule {schedule_name} created with:\n"
"\tschedule_time=datetime.time(hour={hour}, minute={minute}, ...)."
"Since this is a hourly schedule, the hour parameter will be ignored and the schedule "
"will run on the {minute} mark for the previous hour interval. Replace "
"datetime.time(hour={hour}, minute={minute}, ...) with "
"datetime.time(minute={minute}, ...) to fix this warning.")
cron_schedule = '{minute} * * * *'.format(minute=execution_time.minute)
partition_fn = date_partition_range(start_date,
end=end_date,
delta=datetime.timedelta(hours=1),
fmt="%Y-%m-%d-%H:%M")
def inner(fn):
check.callable_param(fn, 'fn')
schedule_name = name or fn.__name__
tags_fn_for_partition_value = lambda partition: {}
if tags_fn_for_date:
tags_fn_for_partition_value = lambda partition: tags_fn_for_date(
partition.value)
partition_set = PartitionSetDefinition(
name='{}_partitions'.format(schedule_name),
pipeline_name=pipeline_name,
partition_fn=partition_fn,
run_config_fn_for_partition=lambda partition: fn(partition.value),
solid_selection=solid_selection,
tags_fn_for_partition=tags_fn_for_partition_value,
mode=mode,
)
return partition_set.create_schedule_definition(
schedule_name,
cron_schedule,
should_execute=should_execute,
environment_vars=environment_vars,
)
return inner
def __init__(self, base_dir, inst_data=None):
self._base_dir = base_dir
self._subscription_manager = LocalComputeLogSubscriptionManager(self)
self._inst_data = check.opt_inst_param(inst_data, 'inst_data',
ConfigurableClassData)
def __init__(
self,
host="localhost",
port=None,
socket=None,
max_workers=1,
loadable_target_origin=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
ipc_output_file=None,
fixed_server_id=None,
):
check.opt_str_param(host, "host")
check.opt_int_param(port, "port")
check.opt_str_param(socket, "socket")
check.int_param(max_workers, "max_workers")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin",
LoadableTargetOrigin)
check.invariant(
port is not None if seven.IS_WINDOWS else True,
"You must pass a valid `port` on Windows: `socket` not supported.",
)
check.invariant(
(port or socket) and not (port and socket),
"You must pass one and only one of `port` or `socket`.",
)
check.invariant(
host is not None if port else True,
"Must provide a host when serving on a port",
)
check.bool_param(heartbeat, "heartbeat")
check.int_param(heartbeat_timeout, "heartbeat_timeout")
self._ipc_output_file = check.opt_str_param(ipc_output_file,
"ipc_output_file")
check.opt_str_param(fixed_server_id, "fixed_server_id")
check.invariant(heartbeat_timeout > 0,
"heartbeat_timeout must be greater than 0")
check.invariant(
max_workers > 1 if heartbeat else True,
"max_workers must be greater than 1 if heartbeat is True",
)
self.server = grpc.server(ThreadPoolExecutor(max_workers=max_workers))
self._server_termination_event = threading.Event()
self._api_servicer = DagsterApiServer(
server_termination_event=self._server_termination_event,
loadable_target_origin=loadable_target_origin,
heartbeat=heartbeat,
heartbeat_timeout=heartbeat_timeout,
lazy_load_user_code=lazy_load_user_code,
fixed_server_id=fixed_server_id,
)
# Create a health check servicer
self._health_servicer = health.HealthServicer()
health_pb2_grpc.add_HealthServicer_to_server(self._health_servicer,
self.server)
add_DagsterApiServicer_to_server(self._api_servicer, self.server)
if port:
server_address = host + ":" + str(port)
else:
server_address = "unix:" + os.path.abspath(socket)
# grpc.Server.add_insecure_port returns:
# - 0 on failure
# - port number when a port is successfully bound
# - 1 when a UDS is successfully bound
res = self.server.add_insecure_port(server_address)
if socket and res != 1:
if self._ipc_output_file:
with ipc_write_stream(self._ipc_output_file) as ipc_stream:
ipc_stream.send(GrpcServerFailedToBindEvent())
raise CouldNotBindGrpcServerToAddress(socket)
if port and res != port:
if self._ipc_output_file:
with ipc_write_stream(self._ipc_output_file) as ipc_stream:
ipc_stream.send(GrpcServerFailedToBindEvent())
raise CouldNotBindGrpcServerToAddress(port)
def __init__(self, handle, solid, parent):
self.handleID = check.inst_param(handle, 'handle', SolidHandle)
self.solid = check.inst_param(solid, 'solid', DauphinSolid)
self.parent = check.opt_inst_param(parent, 'parent',
DauphinSolidHandle)
def monthly_schedule(
pipeline_name: str,
start_date: datetime.datetime,
name: Optional[str] = None,
execution_day_of_month: int = 1,
execution_time: datetime.time = datetime.time(0, 0),
tags_fn_for_date: Optional[Callable[[datetime.datetime], Optional[Dict[str, str]]]] = None,
solid_selection: Optional[List[str]] = None,
mode: Optional[str] = "default",
should_execute: Optional[Callable[["ScheduleExecutionContext"], bool]] = None,
environment_vars: Optional[Dict[str, str]] = None,
end_date: Optional[datetime.datetime] = None,
execution_timezone: Optional[str] = None,
partition_months_offset: Optional[int] = 1,
description: Optional[str] = None,
) -> Callable[[Callable[[datetime.datetime], Dict[str, Any]]], PartitionScheduleDefinition]:
"""Create a partitioned schedule that runs monthly.
The decorated function should accept a datetime object as its only argument. The datetime
represents the date partition that it's meant to run on.
The decorated function should return a run configuration dictionary, which will be used as
configuration for the scheduled run.
The decorator produces a :py:class:`~dagster.PartitionScheduleDefinition`.
Args:
pipeline_name (str): The name of the pipeline to execute when the schedule runs.
start_date (datetime.datetime): The date from which to run the schedule.
name (Optional[str]): The name of the schedule to create.
execution_day_of_month (int): The day of the month on which to run the schedule (must be
between 0 and 31).
execution_time (datetime.time): The time at which to execute the schedule.
tags_fn_for_date (Optional[Callable[[datetime.datetime], Optional[Dict[str, str]]]]): A
function that generates tags to attach to the schedules runs. Takes the date of the
schedule run and returns a dictionary of tags (string key-value pairs).
solid_selection (Optional[List[str]]): A list of solid subselection (including single
solid names) to execute when the schedule runs. e.g. ``['*some_solid+', 'other_solid']``
mode (Optional[str]): The pipeline mode in which to execute this schedule.
(Default: 'default')
should_execute (Optional[Callable[ScheduleExecutionContext, bool]]): A function that runs at
schedule execution tie to determine whether a schedule should execute or skip. Takes a
:py:class:`~dagster.ScheduleExecutionContext` and returns a boolean (``True`` if the
schedule should execute). Defaults to a function that always returns ``True``.
environment_vars (Optional[Dict[str, str]]): Any environment variables to set when executing
the schedule.
end_date (Optional[datetime.datetime]): The last time to run the schedule to, defaults to
current time.
execution_timezone (Optional[str]): Timezone in which the schedule should run. Only works
with DagsterDaemonScheduler, and must be set when using that scheduler.
partition_months_offset (Optional[int]): How many months back to go when choosing the partition
for a given schedule execution. For example, when partition_months_offset=1, the schedule
that executes during month N will fill in the partition for month N-1.
(Default: 1)
description (Optional[str]): A human-readable description of the schedule.
"""
check.opt_str_param(name, "name")
check.inst_param(start_date, "start_date", datetime.datetime)
check.opt_inst_param(end_date, "end_date", datetime.datetime)
check.opt_callable_param(tags_fn_for_date, "tags_fn_for_date")
check.opt_nullable_list_param(solid_selection, "solid_selection", of_type=str)
mode = check.opt_str_param(mode, "mode", DEFAULT_MODE_NAME)
check.opt_callable_param(should_execute, "should_execute")
check.opt_dict_param(environment_vars, "environment_vars", key_type=str, value_type=str)
check.str_param(pipeline_name, "pipeline_name")
check.int_param(execution_day_of_month, "execution_day")
check.inst_param(execution_time, "execution_time", datetime.time)
check.opt_str_param(execution_timezone, "execution_timezone")
check.opt_int_param(partition_months_offset, "partition_months_offset")
check.opt_str_param(description, "description")
if (
start_date.day != 1
or start_date.hour != 0
or start_date.minute != 0
or start_date.second != 0
):
warnings.warn(
"`start_date` must be at the beginning of the first day of the month for a monthly "
"schedule. Use `execution_day_of_month` and `execution_time` to execute the schedule "
"at a specific time within the month. For example, to run the schedule at 3AM on the "
"23rd of each month starting in October, your schedule definition would look like:"
"""
@monthly_schedule(
start_date=datetime.datetime(2020, 10, 1),
execution_day_of_month=23,
execution_time=datetime.time(3, 0)
):
def my_schedule_definition(_):
...
"""
)
if execution_day_of_month <= 0 or execution_day_of_month > 31:
raise DagsterInvalidDefinitionError(
"`execution_day_of_month={}` is not valid for monthly schedule. Execution day must be "
"between 1 and 31".format(execution_day_of_month)
)
cron_schedule = "{minute} {hour} {day} * *".format(
minute=execution_time.minute, hour=execution_time.hour, day=execution_day_of_month
)
fmt = DEFAULT_MONTHLY_FORMAT
execution_time_to_partition_fn = (
lambda d: pendulum.instance(d)
.replace(hour=0, minute=0)
.subtract(months=partition_months_offset, days=execution_day_of_month - 1)
)
partition_fn = schedule_partition_range(
start_date,
end=end_date,
cron_schedule=cron_schedule,
fmt=fmt,
timezone=execution_timezone,
execution_time_to_partition_fn=execution_time_to_partition_fn,
inclusive=(partition_months_offset == 0),
)
def inner(fn: Callable[[datetime.datetime], Dict[str, Any]]) -> PartitionScheduleDefinition:
check.callable_param(fn, "fn")
schedule_name = name or fn.__name__
tags_fn_for_partition_value: Callable[
["Partition"], Optional[Dict[str, str]]
] = lambda partition: {}
if tags_fn_for_date:
tags_fn = cast(
Callable[[datetime.datetime], Optional[Dict[str, str]]], tags_fn_for_date
)
tags_fn_for_partition_value = lambda partition: tags_fn(partition.value)
partition_set = PartitionSetDefinition(
name="{}_partitions".format(schedule_name),
pipeline_name=pipeline_name,
partition_fn=partition_fn,
run_config_fn_for_partition=lambda partition: fn(partition.value),
solid_selection=solid_selection,
tags_fn_for_partition=tags_fn_for_partition_value,
mode=mode,
)
return partition_set.create_schedule_definition(
schedule_name,
cron_schedule,
should_execute=should_execute,
environment_vars=environment_vars,
partition_selector=create_offset_partition_selector(
execution_time_to_partition_fn=execution_time_to_partition_fn
),
execution_timezone=execution_timezone,
description=description,
)
return inner
def __init__(
self,
origin: RepositoryLocationOrigin,
host: Optional[str] = None,
port: Optional[int] = None,
socket: Optional[str] = None,
server_id: Optional[str] = None,
heartbeat: Optional[bool] = False,
watch_server: Optional[bool] = True,
grpc_server_registry: Optional[GrpcServerRegistry] = None,
):
from dagster.grpc.client import DagsterGrpcClient, client_heartbeat_thread
self._origin = check.inst_param(origin, "origin",
RepositoryLocationOrigin)
self.grpc_server_registry = check.opt_inst_param(
grpc_server_registry, "grpc_server_registry", GrpcServerRegistry)
if isinstance(self.origin, GrpcServerRepositoryLocationOrigin):
self._port = self.origin.port
self._socket = self.origin.socket
self._host = self.origin.host
self._use_ssl = bool(self.origin.use_ssl)
else:
self._port = check.opt_int_param(port, "port")
self._socket = check.opt_str_param(socket, "socket")
self._host = check.str_param(host, "host")
self._use_ssl = False
self._watch_thread_shutdown_event = None
self._watch_thread = None
self._heartbeat_shutdown_event = None
self._heartbeat_thread = None
self._heartbeat = check.bool_param(heartbeat, "heartbeat")
self._watch_server = check.bool_param(watch_server, "watch_server")
self.server_id = None
self._external_repositories_data = None
self._executable_path = None
self._container_image = None
self._container_context = None
self._repository_code_pointer_dict = None
self._entry_point = None
try:
self.client = DagsterGrpcClient(
port=self._port,
socket=self._socket,
host=self._host,
use_ssl=self._use_ssl,
)
list_repositories_response = sync_list_repositories_grpc(
self.client)
self.server_id = server_id if server_id else sync_get_server_id(
self.client)
self.repository_names = set(
symbol.repository_name
for symbol in list_repositories_response.repository_symbols)
if self._heartbeat:
self._heartbeat_shutdown_event = threading.Event()
self._heartbeat_thread = threading.Thread(
target=client_heartbeat_thread,
args=(
self.client,
self._heartbeat_shutdown_event,
),
name="grpc-client-heartbeat",
)
self._heartbeat_thread.daemon = True
self._heartbeat_thread.start()
self._executable_path = list_repositories_response.executable_path
self._repository_code_pointer_dict = (
list_repositories_response.repository_code_pointer_dict)
self._entry_point = list_repositories_response.entry_point
self._container_image = (
list_repositories_response.container_image
or self._reload_current_image(
) # Back-compat for older gRPC servers that did not include container_image in ListRepositoriesResponse
)
self._container_context = list_repositories_response.container_context
self._external_repositories_data = sync_get_streaming_external_repositories_data_grpc(
self.client,
self,
)
self.external_repositories = {
repo_name: ExternalRepository(
repo_data,
RepositoryHandle(
repository_name=repo_name,
repository_location=self,
),
)
for repo_name, repo_data in
self._external_repositories_data.items()
}
except:
self.cleanup()
raise
def check_opt_field_param(obj, param_name):
return check.opt_inst_param(obj, param_name, FieldImpl)
def __init__(self, solid_def, given_alias=None, tags=None):
self.solid_def = solid_def
self.given_alias = check.opt_str_param(given_alias, 'given_alias')
self.tags = check.opt_inst_param(tags, 'tags', frozentags)
def create_run_for_pipeline(
self,
pipeline_def,
execution_plan=None,
run_id=None,
run_config=None,
mode=None,
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
solid_selection=None,
):
from dagster.core.execution.api import create_execution_plan
from dagster.core.execution.plan.plan import ExecutionPlan
from dagster.core.snap import snapshot_from_execution_plan
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
check.opt_inst_param(execution_plan, 'execution_plan', ExecutionPlan)
# note that solids_to_execute is required to execute the solid subset, which is the
# frozenset version of the previous solid_subset.
# solid_selection is not required and will not be converted to solids_to_execute here.
# i.e. this function doesn't handle solid queries.
# solid_selection is only used to pass the user queries further down.
check.opt_set_param(solids_to_execute, 'solids_to_execute', of_type=str)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
if solids_to_execute:
if isinstance(pipeline_def, PipelineSubsetDefinition):
# for the case when pipeline_def is created by ExecutablePipeline or ExternalPipeline
check.invariant(
solids_to_execute == pipeline_def.solids_to_execute,
'Cannot create a PipelineRun from pipeline subset {pipeline_solids_to_execute} '
'that conflicts with solids_to_execute arg {solids_to_execute}'.format(
pipeline_solids_to_execute=str_format_list(pipeline_def.solids_to_execute),
solids_to_execute=str_format_list(solids_to_execute),
),
)
else:
# for cases when `create_run_for_pipeline` is directly called
pipeline_def = pipeline_def.get_pipeline_subset_def(
solids_to_execute=solids_to_execute
)
if execution_plan is None:
execution_plan = create_execution_plan(
pipeline_def,
run_config=run_config,
mode=mode,
step_keys_to_execute=step_keys_to_execute,
)
return self.create_run(
pipeline_name=pipeline_def.name,
run_id=run_id,
run_config=run_config,
mode=check.opt_str_param(mode, 'mode', default=pipeline_def.get_default_mode_name()),
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
step_keys_to_execute=step_keys_to_execute,
status=status,
tags=tags,
root_run_id=root_run_id,
parent_run_id=parent_run_id,
pipeline_snapshot=pipeline_def.get_pipeline_snapshot(),
execution_plan_snapshot=snapshot_from_execution_plan(
execution_plan, pipeline_def.get_pipeline_snapshot_id()
),
parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(),
)
def __new__(cls, input_name: str, type_check_data: TypeCheckData):
return super(StepInputData, cls).__new__(
cls,
input_name=check.str_param(input_name, "input_name"),
type_check_data=check.opt_inst_param(type_check_data, "type_check_data", TypeCheckData),
)
def scoped_pipeline_context(
pipeline_def,
environment_dict,
pipeline_run,
instance,
execution_plan,
system_storage_data=None,
scoped_resources_builder_cm=create_resource_builder,
raise_on_error=False,
):
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
check.dict_param(environment_dict, 'environment_dict', key_type=str)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.opt_inst_param(system_storage_data, 'system_storage_data',
SystemStorageData)
context_creation_data = create_context_creation_data(
pipeline_def,
environment_dict,
pipeline_run,
instance,
execution_plan,
)
# After this try block, a Dagster exception thrown will result in a pipeline init failure event.
pipeline_context = None
try:
executor_config = create_executor_config(context_creation_data)
log_manager = create_log_manager(context_creation_data)
with scoped_resources_builder_cm(
context_creation_data.pipeline_def,
context_creation_data.environment_config,
context_creation_data.pipeline_run,
log_manager,
context_creation_data.resource_keys_to_init,
) as scoped_resources_builder:
system_storage_data = create_system_storage_data(
context_creation_data, system_storage_data,
scoped_resources_builder)
pipeline_context = construct_pipeline_execution_context(
context_creation_data=context_creation_data,
scoped_resources_builder=scoped_resources_builder,
system_storage_data=system_storage_data,
log_manager=log_manager,
executor_config=executor_config,
raise_on_error=raise_on_error,
)
yield pipeline_context
except DagsterError as dagster_error:
# only yield an init failure event if we haven't already yielded context
if pipeline_context is None:
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info
if dagster_error.is_user_code_error else sys.exc_info())
error_info = serializable_error_info_from_exc_info(
user_facing_exc_info)
yield DagsterEvent.pipeline_init_failure(
pipeline_name=pipeline_def.name,
failure_data=PipelineInitFailureData(error=error_info),
log_manager=_create_context_free_log_manager(
instance, pipeline_run, pipeline_def),
)
if raise_on_error:
raise dagster_error
# if we've caught an error after context init we're in a problematic state and should just raise
else:
raise dagster_error
def execute_solid(
solid_def,
mode_def=None,
input_values=None,
tags=None,
run_config=None,
raise_on_error=True,
environment_dict=None,
):
'''Execute a single solid in an ephemeral pipeline.
Intended to support unit tests. Input values may be passed directly, and no pipeline need be
specified -- an ephemeral pipeline will be constructed.
Args:
solid_def (SolidDefinition): The solid to execute.
mode_def (Optional[ModeDefinition]): The mode within which to execute the solid. Use this
if, e.g., custom resources, loggers, or executors are desired.
input_values (Optional[Dict[str, Any]]): A dict of input names to input values, used to
pass inputs to the solid directly. You may also use the ``run_config`` to
configure any inputs that are configurable.
tags (Optional[Dict[str, Any]]): Arbitrary key-value pairs that will be added to pipeline
logs.
run_config (Optional[dict]): The environment configuration that parameterized this
execution, as a dict.
raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur.
Defaults to ``True``, since this is the most useful behavior in test.
Returns:
Union[CompositeSolidExecutionResult, SolidExecutionResult]: The result of executing the
solid.
'''
check.inst_param(solid_def, 'solid_def', ISolidDefinition)
check.opt_inst_param(mode_def, 'mode_def', ModeDefinition)
input_values = check.opt_dict_param(input_values,
'input_values',
key_type=str)
# backcompact
run_config = canonicalize_run_config(run_config, environment_dict)
solid_defs = [solid_def]
def create_value_solid(input_name, input_value):
@lambda_solid(name=input_name)
def input_solid():
return input_value
return input_solid
dependencies = defaultdict(dict)
for input_name, input_value in input_values.items():
dependencies[solid_def.name][input_name] = DependencyDefinition(
input_name)
solid_defs.append(create_value_solid(input_name, input_value))
result = execute_pipeline(
PipelineDefinition(
name='ephemeral_{}_solid_pipeline'.format(solid_def.name),
solid_defs=solid_defs,
dependencies=dependencies,
mode_defs=[mode_def] if mode_def else None,
),
run_config=run_config,
mode=mode_def.name if mode_def else None,
tags=tags,
raise_on_error=raise_on_error,
)
return result.result_for_handle(solid_def.name)
def create_run_for_pipeline(
self,
pipeline_def,
execution_plan=None,
run_id=None,
run_config=None,
mode=None,
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
solid_selection=None,
):
from dagster.core.execution.api import create_execution_plan
from dagster.core.execution.plan.plan import ExecutionPlan
from dagster.core.snap import snapshot_from_execution_plan
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan)
# note that solids_to_execute is required to execute the solid subset, which is the
# frozenset version of the previous solid_subset.
# solid_selection is not required and will not be converted to solids_to_execute here.
# i.e. this function doesn't handle solid queries.
# solid_selection is only used to pass the user queries further down.
check.opt_set_param(solids_to_execute,
"solids_to_execute",
of_type=str)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
if solids_to_execute:
if isinstance(pipeline_def, PipelineSubsetDefinition):
# for the case when pipeline_def is created by IPipeline or ExternalPipeline
check.invariant(
solids_to_execute == pipeline_def.solids_to_execute,
"Cannot create a PipelineRun from pipeline subset {pipeline_solids_to_execute} "
"that conflicts with solids_to_execute arg {solids_to_execute}"
.format(
pipeline_solids_to_execute=str_format_list(
pipeline_def.solids_to_execute),
solids_to_execute=str_format_list(solids_to_execute),
),
)
else:
# for cases when `create_run_for_pipeline` is directly called
pipeline_def = pipeline_def.get_pipeline_subset_def(
solids_to_execute=solids_to_execute)
full_execution_plan = execution_plan or create_execution_plan(
pipeline_def,
run_config=run_config,
mode=mode,
)
check.invariant(
len(full_execution_plan.step_keys_to_execute) == len(
full_execution_plan.steps))
if _is_memoized_run(tags):
if step_keys_to_execute:
raise DagsterInvariantViolationError(
"step_keys_to_execute parameter cannot be used in conjunction with memoized "
"pipeline runs.")
step_keys_to_execute = self.resolve_unmemoized_steps(
full_execution_plan,
run_config=run_config,
mode=mode,
) # TODO: tighter integration with existing step_keys_to_execute functionality
subsetted_execution_plan = (
full_execution_plan.build_subset_plan(step_keys_to_execute)
if step_keys_to_execute else full_execution_plan)
return self.create_run(
pipeline_name=pipeline_def.name,
run_id=run_id,
run_config=run_config,
mode=check.opt_str_param(
mode, "mode", default=pipeline_def.get_default_mode_name()),
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
step_keys_to_execute=step_keys_to_execute,
status=status,
tags=tags,
root_run_id=root_run_id,
parent_run_id=parent_run_id,
pipeline_snapshot=pipeline_def.get_pipeline_snapshot(),
execution_plan_snapshot=snapshot_from_execution_plan(
subsetted_execution_plan,
pipeline_def.get_pipeline_snapshot_id()),
parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(
),
)
def launch_scheduled_runs_for_schedule(
instance,
logger,
external_schedule: ExternalSchedule,
schedule_state: InstigatorState,
workspace,
end_datetime_utc: datetime.datetime,
max_catchup_runs,
max_tick_retries,
debug_crash_flags=None,
log_verbose_checks=True,
):
instance = check.inst_param(instance, "instance", DagsterInstance)
schedule_state = check.opt_inst_param(schedule_state, "schedule_state", InstigatorState)
end_datetime_utc = check.inst_param(end_datetime_utc, "end_datetime_utc", datetime.datetime)
instigator_origin_id = external_schedule.get_external_origin_id()
ticks = instance.get_ticks(instigator_origin_id, limit=1)
latest_tick = ticks[0] if ticks else None
instigator_data = cast(ScheduleInstigatorData, schedule_state.instigator_data)
start_timestamp_utc = instigator_data.start_timestamp if schedule_state else None
if latest_tick:
if latest_tick.status == TickStatus.STARTED or (
latest_tick.status == TickStatus.FAILURE
and latest_tick.failure_count <= max_tick_retries
):
# Scheduler was interrupted while performing this tick, re-do it
start_timestamp_utc = (
max(start_timestamp_utc, latest_tick.timestamp)
if start_timestamp_utc
else latest_tick.timestamp
)
else:
start_timestamp_utc = (
max(start_timestamp_utc, latest_tick.timestamp + 1)
if start_timestamp_utc
else latest_tick.timestamp + 1
)
else:
start_timestamp_utc = instigator_data.start_timestamp
schedule_name = external_schedule.name
timezone_str = external_schedule.execution_timezone
if not timezone_str:
timezone_str = "UTC"
if log_verbose_checks:
logger.warn(
f"Using UTC as the timezone for {external_schedule.name} as it did not specify "
"an execution_timezone in its definition."
)
tick_times = []
for next_time in external_schedule.execution_time_iterator(start_timestamp_utc):
if next_time.timestamp() > end_datetime_utc.timestamp():
break
tick_times.append(next_time)
if not tick_times:
if log_verbose_checks:
logger.info(f"No new runs for {schedule_name}")
return
if not external_schedule.partition_set_name and len(tick_times) > 1:
logger.warning(f"{schedule_name} has no partition set, so not trying to catch up")
tick_times = tick_times[-1:]
elif len(tick_times) > max_catchup_runs:
logger.warning(f"{schedule_name} has fallen behind, only launching {max_catchup_runs} runs")
tick_times = tick_times[-max_catchup_runs:]
if len(tick_times) == 1:
tick_time = tick_times[0].strftime(default_date_format_string())
logger.info(f"Evaluating schedule `{schedule_name}` at {tick_time}")
else:
times = ", ".join([time.strftime(default_date_format_string()) for time in tick_times])
logger.info(f"Evaluating schedule `{schedule_name}` at the following times: {times}")
for schedule_time in tick_times:
schedule_timestamp = schedule_time.timestamp()
schedule_time_str = schedule_time.strftime(default_date_format_string())
if latest_tick and latest_tick.timestamp == schedule_timestamp:
tick = latest_tick
if latest_tick.status == TickStatus.FAILURE:
logger.info(f"Retrying previously failed schedule execution at {schedule_time_str}")
else:
logger.info(
f"Resuming previously interrupted schedule execution at {schedule_time_str}"
)
else:
tick = instance.create_tick(
TickData(
instigator_origin_id=instigator_origin_id,
instigator_name=schedule_name,
instigator_type=InstigatorType.SCHEDULE,
status=TickStatus.STARTED,
timestamp=schedule_timestamp,
)
)
_check_for_debug_crash(debug_crash_flags, "TICK_CREATED")
with _ScheduleLaunchContext(tick, instance, logger) as tick_context:
try:
_check_for_debug_crash(debug_crash_flags, "TICK_HELD")
yield from _schedule_runs_at_time(
instance,
logger,
workspace,
external_schedule,
schedule_time,
tick_context,
debug_crash_flags,
)
except Exception as e:
if isinstance(e, DagsterUserCodeUnreachableError):
try:
raise DagsterSchedulerError(
f"Unable to reach the user code server for schedule {schedule_name}. Schedule will resume execution once the server is available."
) from e
except:
error_data = serializable_error_info_from_exc_info(sys.exc_info())
tick_context.update_state(
TickStatus.FAILURE,
error=error_data,
# don't increment the failure count - retry forever until the server comes back up
# or the schedule is turned off
failure_count=tick_context.failure_count,
)
raise # Raise the wrapped DagsterSchedulerError exception
else:
error_data = serializable_error_info_from_exc_info(sys.exc_info())
tick_context.update_state(
TickStatus.FAILURE,
error=error_data,
failure_count=tick_context.failure_count + 1,
)
raise
def __init__(self, conn_string, inst_data=None):
check.str_param(conn_string, 'conn_string')
self.engine = create_engine(conn_string)
self._inst_data = check.opt_inst_param(inst_data, 'inst_data',
ConfigurableClassData)
def _check_execute_pipeline_args(pipeline,
run_config,
mode,
preset,
tags,
instance,
solid_selection=None):
pipeline = _check_pipeline(pipeline)
pipeline_def = pipeline.get_definition()
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config = check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
check.opt_str_param(preset, "preset")
check.invariant(
not (mode is not None and preset is not None),
"You may set only one of `mode` (got {mode}) or `preset` (got {preset})."
.format(mode=mode, preset=preset),
)
tags = check.opt_dict_param(tags, "tags", key_type=str)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
if preset is not None:
pipeline_preset = pipeline_def.get_preset(preset)
if pipeline_preset.run_config is not None:
check.invariant(
(not run_config) or (pipeline_preset.run_config == run_config),
"The environment set in preset '{preset}' does not agree with the environment "
"passed in the `run_config` argument.".format(preset=preset),
)
run_config = pipeline_preset.run_config
# load solid_selection from preset
if pipeline_preset.solid_selection is not None:
check.invariant(
solid_selection is None
or solid_selection == pipeline_preset.solid_selection,
"The solid_selection set in preset '{preset}', {preset_subset}, does not agree with "
"the `solid_selection` argument: {solid_selection}".format(
preset=preset,
preset_subset=pipeline_preset.solid_selection,
solid_selection=solid_selection,
),
)
solid_selection = pipeline_preset.solid_selection
check.invariant(
mode is None or mode == pipeline_preset.mode,
"Mode {mode} does not agree with the mode set in preset '{preset}': "
"('{preset_mode}')".format(preset=preset,
preset_mode=pipeline_preset.mode,
mode=mode),
)
mode = pipeline_preset.mode
tags = merge_dicts(pipeline_preset.tags, tags)
if mode is not None:
if not pipeline_def.has_mode_definition(mode):
raise DagsterInvariantViolationError((
"You have attempted to execute pipeline {name} with mode {mode}. "
"Available modes: {modes}").format(
name=pipeline_def.name,
mode=mode,
modes=pipeline_def.available_modes,
))
else:
if pipeline_def.is_multi_mode:
raise DagsterInvariantViolationError((
"Pipeline {name} has multiple modes (Available modes: {modes}) and you have "
"attempted to execute it without specifying a mode. Set "
"mode property on the PipelineRun object.").format(
name=pipeline_def.name,
modes=pipeline_def.available_modes))
mode = pipeline_def.get_default_mode_name()
tags = merge_dicts(pipeline_def.tags, tags)
check.opt_inst_param(instance, "instance", DagsterInstance)
instance = instance or DagsterInstance.ephemeral()
# generate pipeline subset from the given solid_selection
if solid_selection:
pipeline = pipeline.subset_for_execution(solid_selection)
return (
pipeline,
run_config,
instance,
mode,
tags,
pipeline.solids_to_execute,
solid_selection,
)
def __init__(
self,
solid_defs,
name=None,
description=None,
dependencies=None,
mode_defs=None,
preset_defs=None,
tags=None,
hook_defs=None,
input_mappings=None,
output_mappings=None,
config_mapping=None,
positional_inputs=None,
_parent_pipeline_def=None, # https://github.com/dagster-io/dagster/issues/2115
):
if not name:
warnings.warn(
"Pipeline must have a name. Names will be required starting in 0.10.0 or later."
)
name = _anonymous_pipeline_name()
# For these warnings they check truthiness because they get changed to [] higher
# in the stack for the decorator case
if input_mappings:
experimental_arg_warning("input_mappings", "PipelineDefinition")
if output_mappings:
experimental_arg_warning("output_mappings", "PipelineDefinition")
if config_mapping is not None:
experimental_arg_warning("config_mapping", "PipelineDefinition")
if positional_inputs:
experimental_arg_warning("positional_inputs", "PipelineDefinition")
super(PipelineDefinition, self).__init__(
name=name,
description=description,
dependencies=dependencies,
node_defs=solid_defs,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
input_mappings=input_mappings,
output_mappings=output_mappings,
config_mapping=config_mapping,
)
self._current_level_node_defs = solid_defs
self._tags = validate_tags(tags)
mode_definitions = check.opt_list_param(mode_defs,
"mode_defs",
of_type=ModeDefinition)
if not mode_definitions:
mode_definitions = [ModeDefinition()]
self._mode_definitions = mode_definitions
seen_modes = set()
for mode_def in mode_definitions:
if mode_def.name in seen_modes:
raise DagsterInvalidDefinitionError((
'Two modes seen with the name "{mode_name}" in "{pipeline_name}". '
"Modes must have unique names.").format(
mode_name=mode_def.name, pipeline_name=self._name))
seen_modes.add(mode_def.name)
self._dagster_type_dict = construct_dagster_type_dictionary(
self._current_level_node_defs)
self._hook_defs = check.opt_set_param(hook_defs,
"hook_defs",
of_type=HookDefinition)
self._preset_defs = check.opt_list_param(preset_defs, "preset_defs",
PresetDefinition)
self._preset_dict = {}
for preset in self._preset_defs:
if preset.name in self._preset_dict:
raise DagsterInvalidDefinitionError((
'Two PresetDefinitions seen with the name "{name}" in "{pipeline_name}". '
"PresetDefinitions must have unique names.").format(
name=preset.name, pipeline_name=self._name))
if preset.mode not in seen_modes:
raise DagsterInvalidDefinitionError(
('PresetDefinition "{name}" in "{pipeline_name}" '
'references mode "{mode}" which is not defined.').format(
name=preset.name,
pipeline_name=self._name,
mode=preset.mode))
self._preset_dict[preset.name] = preset
# Validate solid resource dependencies
_validate_resource_dependencies(
self._mode_definitions,
self._current_level_node_defs,
self._dagster_type_dict,
self._solid_dict,
self._hook_defs,
)
# Validate unsatisfied inputs can be materialized from config
_validate_inputs(self._dependency_structure, self._solid_dict)
# Recursively explore all nodes in the this pipeline
self._all_node_defs = _build_all_node_defs(
self._current_level_node_defs)
self._parent_pipeline_def = check.opt_inst_param(
_parent_pipeline_def, "_parent_pipeline_def", PipelineDefinition)
self._cached_run_config_schemas = {}
self._cached_external_pipeline = None
def get_context(self, solid_config=None, mode_def=None, environment_dict=None):
'''Get a dagstermill execution context for interactive exploration and development.
Args:
solid_config (Optional[Any]): If specified, this value will be made available on the
context as its ``solid_config`` property.
mode_def (Optional[:class:`dagster.ModeDefinition`]): If specified, defines the mode to
use to construct the context. Specify this if you would like a context constructed
with specific ``resource_defs`` or ``logger_defs``. By default, an ephemeral mode
with a console logger will be constructed.
environment_dict(Optional[dict]): The environment config dict with which to construct
the context.
Returns:
:py:class:`~dagstermill.DagstermillExecutionContext`
'''
check.opt_inst_param(mode_def, 'mode_def', ModeDefinition)
environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
# If we are running non-interactively, and there is already a context reconstituted, return
# that context rather than overwriting it.
if self.context is not None and isinstance(
self.context, DagstermillRuntimeExecutionContext
):
return self.context
if not mode_def:
mode_def = ModeDefinition(logger_defs={'dagstermill': colored_console_logger})
environment_dict['loggers'] = {'dagstermill': {}}
solid_def = SolidDefinition(
name='this_solid',
input_defs=[],
compute_fn=lambda *args, **kwargs: None,
output_defs=[],
description='Ephemeral solid constructed by dagstermill.get_context()',
required_resource_keys=mode_def.resource_key_set,
)
pipeline_def = PipelineDefinition(
[solid_def], mode_defs=[mode_def], name='ephemeral_dagstermill_pipeline'
)
run_id = make_new_run_id()
# construct stubbed PipelineRun for notebook exploration...
# The actual pipeline run during pipeline execution will be serialized and reconstituted
# in the `reconstitute_pipeline_context` call
pipeline_run = PipelineRun(
pipeline_name=pipeline_def.name,
run_id=run_id,
environment_dict=environment_dict,
mode=mode_def.name,
step_keys_to_execute=None,
status=PipelineRunStatus.NOT_STARTED,
tags=None,
)
self.in_pipeline = False
self.solid_def = solid_def
self.pipeline = pipeline_def
execution_plan = create_execution_plan(self.pipeline, environment_dict, mode=mode_def.name)
with scoped_pipeline_context(
execution_plan,
environment_dict,
pipeline_run,
DagsterInstance.ephemeral(),
scoped_resources_builder_cm=self._setup_resources,
) as pipeline_context:
self.context = DagstermillExecutionContext(
pipeline_context=pipeline_context,
solid_config=solid_config,
resource_keys_to_init=get_required_resource_keys_to_init(
execution_plan, pipeline_context.system_storage_def
),
)
return self.context
def _execute_step_k8s_job(
_self,
instance_ref_dict,
step_keys,
run_config,
mode,
repo_name,
repo_location_name,
run_id,
job_config_dict,
job_namespace,
load_incluster_config,
retries_dict,
pipeline_origin_packed,
user_defined_k8s_config_dict=None,
kubeconfig_file=None,
):
"""Run step execution in a K8s job pod.
"""
check.dict_param(instance_ref_dict, "instance_ref_dict")
check.list_param(step_keys, "step_keys", of_type=str)
check.invariant(
len(step_keys) == 1, "Celery K8s task executor can only execute 1 step at a time"
)
check.dict_param(run_config, "run_config")
check.str_param(mode, "mode")
check.str_param(repo_name, "repo_name")
check.str_param(repo_location_name, "repo_location_name")
check.str_param(run_id, "run_id")
# Celery will serialize this as a list
job_config = DagsterK8sJobConfig.from_dict(job_config_dict)
check.inst_param(job_config, "job_config", DagsterK8sJobConfig)
check.str_param(job_namespace, "job_namespace")
check.bool_param(load_incluster_config, "load_incluster_config")
check.dict_param(retries_dict, "retries_dict")
pipeline_origin = unpack_value(
check.dict_param(
pipeline_origin_packed, "pipeline_origin_packed"
) # TODO: make part of args
)
check.inst(pipeline_origin, PipelineOrigin)
user_defined_k8s_config = UserDefinedDagsterK8sConfig.from_dict(
user_defined_k8s_config_dict
)
check.opt_inst_param(
user_defined_k8s_config, "user_defined_k8s_config", UserDefinedDagsterK8sConfig,
)
check.opt_str_param(kubeconfig_file, "kubeconfig_file")
# For when launched via DinD or running the cluster
if load_incluster_config:
kubernetes.config.load_incluster_config()
else:
kubernetes.config.load_kube_config(kubeconfig_file)
instance_ref = InstanceRef.from_dict(instance_ref_dict)
instance = DagsterInstance.from_ref(instance_ref)
pipeline_run = instance.get_run_by_id(run_id)
check.invariant(pipeline_run, "Could not load run {}".format(run_id))
step_key = step_keys[0]
if pipeline_run.status != PipelineRunStatus.STARTED:
instance.report_engine_event(
"Not scheduling step because pipeline run status is not STARTED",
pipeline_run,
EngineEventData([EventMetadataEntry.text(step_key, "Step keys"),]),
CeleryK8sJobExecutor,
step_key=step_key,
)
return
# Ensure we stay below k8s name length limits
k8s_name_key = get_k8s_job_name(run_id, step_key)
retries = Retries.from_config(retries_dict)
if retries.get_attempt_count(step_key):
attempt_number = retries.get_attempt_count(step_key)
job_name = "dagster-job-%s-%d" % (k8s_name_key, attempt_number)
pod_name = "dagster-job-%s-%d" % (k8s_name_key, attempt_number)
else:
job_name = "dagster-job-%s" % (k8s_name_key)
pod_name = "dagster-job-%s" % (k8s_name_key)
input_json = serialize_dagster_namedtuple(
ExecuteStepArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=run_id,
instance_ref=None,
mode=mode,
step_keys_to_execute=step_keys,
run_config=run_config,
retries_dict=retries_dict,
)
)
command = ["dagster"]
args = ["api", "execute_step_with_structured_logs", input_json]
job = construct_dagster_k8s_job(
job_config, command, args, job_name, user_defined_k8s_config, pod_name
)
# Running list of events generated from this task execution
events = []
# Post event for starting execution
job_name = job.metadata.name
engine_event = instance.report_engine_event(
"Executing step {} in Kubernetes job {}".format(step_key, job_name),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_key, "Step keys"),
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
EventMetadataEntry.text(pod_name, "Kubernetes Pod name"),
EventMetadataEntry.text(job_config.job_image, "Job image"),
EventMetadataEntry.text(job_config.image_pull_policy, "Image pull policy"),
EventMetadataEntry.text(
str(job_config.image_pull_secrets), "Image pull secrets"
),
EventMetadataEntry.text(
str(job_config.service_account_name), "Service account name"
),
],
marker_end=DELEGATE_MARKER,
),
CeleryK8sJobExecutor,
# validated above that step_keys is length 1, and it is not possible to use ETH or
# execution plan in this function (Celery K8s workers should not access to user code)
step_key=step_key,
)
events.append(engine_event)
try:
kubernetes.client.BatchV1Api().create_namespaced_job(body=job, namespace=job_namespace)
except kubernetes.client.rest.ApiException as e:
if e.reason == "Conflict":
# There is an existing job with the same name so do not procede.
instance.report_engine_event(
"Did not create Kubernetes job {} for step {} since job name already "
"exists, exiting.".format(job_name, step_key),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_key, "Step keys"),
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
EventMetadataEntry.text(pod_name, "Kubernetes Pod name"),
],
marker_end=DELEGATE_MARKER,
),
CeleryK8sJobExecutor,
step_key=step_key,
)
else:
instance.report_engine_event(
"Encountered unexpected error while creating Kubernetes job {} for step {}, "
"exiting.".format(job_name, step_key),
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_key, "Step keys"),
EventMetadataEntry.text(e, "Error"),
]
),
CeleryK8sJobExecutor,
step_key=step_key,
)
return
try:
wait_for_job_success(
job_name=job_name, namespace=job_namespace, instance=instance, run_id=run_id,
)
except DagsterK8sPipelineStatusException:
instance.report_engine_event(
"Terminating Kubernetes Job because pipeline run status is not STARTED",
pipeline_run,
EngineEventData(
[
EventMetadataEntry.text(step_key, "Step keys"),
EventMetadataEntry.text(job_name, "Kubernetes Job name"),
EventMetadataEntry.text(job_namespace, "Kubernetes Job namespace"),
]
),
CeleryK8sJobExecutor,
step_key=step_key,
)
delete_job(job_name=job_name, namespace=job_namespace)
return
pod_names = get_pod_names_in_job(job_name, namespace=job_namespace)
# Post engine event for log retrieval
engine_event = instance.report_engine_event(
"Retrieving logs from Kubernetes Job pods",
pipeline_run,
EngineEventData([EventMetadataEntry.text("\n".join(pod_names), "Pod names")]),
CeleryK8sJobExecutor,
step_key=step_key,
)
events.append(engine_event)
logs = []
for pod_name in pod_names:
raw_logs = retrieve_pod_logs(pod_name, namespace=job_namespace)
logs += raw_logs.split("\n")
events += filter_dagster_events_from_pod_logs(logs)
serialized_events = [serialize_dagster_namedtuple(event) for event in events]
return serialized_events
