def run_loop(
self,
daemon_uuid,
daemon_shutdown_event,
gen_workspace,
heartbeat_interval_seconds,
error_interval_seconds,
until=None,
):
# Each loop runs in its own thread with its own instance and IWorkspace
with DagsterInstance.get() as instance:
with gen_workspace(instance) as workspace:
check.inst_param(workspace, "workspace", IWorkspace)
while not daemon_shutdown_event.is_set() and (
not until or pendulum.now("UTC") < until):
curr_time = pendulum.now("UTC")
if (not self._last_iteration_time
or (curr_time - self._last_iteration_time
).total_seconds() >= self.interval_seconds):
self._last_iteration_time = curr_time
self._run_iteration(
instance,
daemon_uuid,
daemon_shutdown_event,
workspace,
heartbeat_interval_seconds,
error_interval_seconds,
until,
)
self._check_add_heartbeat(instance, daemon_uuid,
heartbeat_interval_seconds,
error_interval_seconds)
daemon_shutdown_event.wait(0.5)
def import_command(input_files: Tuple[str, ...]):
debug_payloads = []
for input_file in input_files:
with GzipFile(input_file, "rb") as file:
blob = file.read().decode("utf-8")
debug_payload = deserialize_as(blob, DebugRunPayload)
debug_payloads.append(debug_payload)
with DagsterInstance.get() as instance:
for debug_payload in debug_payloads:
run = debug_payload.pipeline_run
click.echo(f"Importing run {run.run_id} (Dagster: {debug_payload.version})")
if not instance.has_snapshot(run.execution_plan_snapshot_id):
instance.add_snapshot(
debug_payload.execution_plan_snapshot,
run.execution_plan_snapshot_id,
)
if not instance.has_snapshot(run.pipeline_snapshot_id):
instance.add_snapshot(
debug_payload.pipeline_snapshot,
run.pipeline_snapshot_id,
)
if not instance.has_run(run.run_id):
instance.add_run(run)
for event in tqdm(debug_payload.event_list):
instance.store_event(event)
def test_script_execution(
dagster_instance_with_k8s_scheduler,
unset_dagster_home,
helm_namespace_for_k8s_run_launcher,
restore_k8s_cron_tab,
): # pylint:disable=unused-argument,redefined-outer-name
with tempfile.TemporaryDirectory() as tempdir:
with environ({"DAGSTER_HOME": tempdir}):
local_instance = DagsterInstance.get()
with get_test_external_repo() as external_repo:
# Initialize scheduler
dagster_instance_with_k8s_scheduler.reconcile_scheduler_state(
external_repo)
dagster_instance_with_k8s_scheduler.start_schedule_and_update_storage_state(
external_repo.get_external_schedule(
"no_config_pipeline_every_min_schedule"))
local_runs = local_instance.get_runs()
assert len(local_runs) == 0
cron_job_name = external_repo.get_external_schedule(
"no_config_pipeline_every_min_schedule"
).get_external_origin_id()
batch_v1beta1_api = kubernetes.client.BatchV1beta1Api()
cron_job = batch_v1beta1_api.read_namespaced_cron_job(
cron_job_name, helm_namespace_for_k8s_run_launcher)
container = cron_job.spec.job_template.spec.template.spec.containers[
0]
args = container.args
cli_cmd = [sys.executable, "-m"] + args + [
"--override-system-timezone=US/Eastern"
]
p = subprocess.Popen(
cli_cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
"DAGSTER_HOME": tempdir,
"LC_ALL": "C.UTF-8",
"LANG": "C.UTF-8",
}, # https://stackoverflow.com/questions/36651680/click-will-abort-further-execution-because-python-3-was-configured-to-use-ascii
)
stdout, stderr = p.communicate()
print("Command completed with stdout: ", stdout) # pylint: disable=print-call
print("Command completed with stderr: ", stderr) # pylint: disable=print-call
assert p.returncode == 0
local_runs = local_instance.get_runs()
assert len(local_runs) == 1
run_id = local_runs[0].run_id
pipeline_run = local_instance.get_run_by_id(run_id)
assert pipeline_run
assert pipeline_run.status == PipelineRunStatus.SUCCESS
def default_app():
instance = DagsterInstance.get()
process_context = get_workspace_process_context_from_kwargs(
instance=instance,
version=__version__,
read_only=False,
kwargs={},
)
return create_app(process_context, app_path_prefix="", debug=True)
def export_command(run_id, output_file):
with DagsterInstance.get() as instance:
run = instance.get_run_by_id(run_id)
if run is None:
raise click.UsageError(
"Could not find run with run_id '{}'.\n{}".format(
run_id, _recent_failed_runs_text(instance)))
export_run(instance, run, output_file)
def default_app(debug=False):
instance = DagsterInstance.get()
process_context = get_workspace_process_context_from_kwargs(
instance=instance,
version=__version__,
read_only=False,
kwargs={},
)
return DagitWebserver(process_context, ).create_asgi_app(debug=debug)
def run_loop(self, daemon_uuid, daemon_shutdown_event):
# Each loop runs in its own thread with its own instance
with DagsterInstance.get() as instance:
while not daemon_shutdown_event.is_set():
curr_time = pendulum.now("UTC")
if (not self._last_iteration_time or
(curr_time - self._last_iteration_time).total_seconds() >=
self.interval_seconds):
self._last_iteration_time = curr_time
self._run_iteration(instance, curr_time, daemon_uuid)
daemon_shutdown_event.wait(0.5)
def export_command(run_id, output_file):
with DagsterInstance.get() as instance:
run = instance.get_run_by_id(run_id)
if run is None:
raise click.UsageError(
"Could not find run with run_id '{}'.\n{}".format(
run_id, _recent_failed_runs_text(instance)))
debug_payload = DebugRunPayload.build(instance, run)
with GzipFile(output_file, "wb") as file:
click.echo("Exporting run_id '{}' to gzip output file {}.".format(
run_id, output_file))
debug_payload.write(file)
def run_command(interval_seconds, max_concurrent_runs):
coordinator = QueuedRunCoordinatorDaemon(
DagsterInstance.get(), max_concurrent_runs=max_concurrent_runs)
click.echo("Starting run coordinator")
coordinator.run(interval_seconds=interval_seconds)
def script_example():
with instance_for_test():
# start_script
execute_pipeline(predict_color, instance=DagsterInstance.get())
