def run_loop( self, daemon_uuid, daemon_shutdown_event, gen_workspace, heartbeat_interval_seconds, error_interval_seconds, until=None, ): # Each loop runs in its own thread with its own instance and IWorkspace with DagsterInstance.get() as instance: with gen_workspace(instance) as workspace: check.inst_param(workspace, "workspace", IWorkspace) while not daemon_shutdown_event.is_set() and ( not until or pendulum.now("UTC") < until): curr_time = pendulum.now("UTC") if (not self._last_iteration_time or (curr_time - self._last_iteration_time ).total_seconds() >= self.interval_seconds): self._last_iteration_time = curr_time self._run_iteration( instance, daemon_uuid, daemon_shutdown_event, workspace, heartbeat_interval_seconds, error_interval_seconds, until, ) self._check_add_heartbeat(instance, daemon_uuid, heartbeat_interval_seconds, error_interval_seconds) daemon_shutdown_event.wait(0.5)
def import_command(input_files: Tuple[str, ...]): debug_payloads = [] for input_file in input_files: with GzipFile(input_file, "rb") as file: blob = file.read().decode("utf-8") debug_payload = deserialize_as(blob, DebugRunPayload) debug_payloads.append(debug_payload) with DagsterInstance.get() as instance: for debug_payload in debug_payloads: run = debug_payload.pipeline_run click.echo(f"Importing run {run.run_id} (Dagster: {debug_payload.version})") if not instance.has_snapshot(run.execution_plan_snapshot_id): instance.add_snapshot( debug_payload.execution_plan_snapshot, run.execution_plan_snapshot_id, ) if not instance.has_snapshot(run.pipeline_snapshot_id): instance.add_snapshot( debug_payload.pipeline_snapshot, run.pipeline_snapshot_id, ) if not instance.has_run(run.run_id): instance.add_run(run) for event in tqdm(debug_payload.event_list): instance.store_event(event)
def test_script_execution( dagster_instance_with_k8s_scheduler, unset_dagster_home, helm_namespace_for_k8s_run_launcher, restore_k8s_cron_tab, ): # pylint:disable=unused-argument,redefined-outer-name with tempfile.TemporaryDirectory() as tempdir: with environ({"DAGSTER_HOME": tempdir}): local_instance = DagsterInstance.get() with get_test_external_repo() as external_repo: # Initialize scheduler dagster_instance_with_k8s_scheduler.reconcile_scheduler_state( external_repo) dagster_instance_with_k8s_scheduler.start_schedule_and_update_storage_state( external_repo.get_external_schedule( "no_config_pipeline_every_min_schedule")) local_runs = local_instance.get_runs() assert len(local_runs) == 0 cron_job_name = external_repo.get_external_schedule( "no_config_pipeline_every_min_schedule" ).get_external_origin_id() batch_v1beta1_api = kubernetes.client.BatchV1beta1Api() cron_job = batch_v1beta1_api.read_namespaced_cron_job( cron_job_name, helm_namespace_for_k8s_run_launcher) container = cron_job.spec.job_template.spec.template.spec.containers[ 0] args = container.args cli_cmd = [sys.executable, "-m"] + args + [ "--override-system-timezone=US/Eastern" ] p = subprocess.Popen( cli_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={ "DAGSTER_HOME": tempdir, "LC_ALL": "C.UTF-8", "LANG": "C.UTF-8", }, # https://stackoverflow.com/questions/36651680/click-will-abort-further-execution-because-python-3-was-configured-to-use-ascii ) stdout, stderr = p.communicate() print("Command completed with stdout: ", stdout) # pylint: disable=print-call print("Command completed with stderr: ", stderr) # pylint: disable=print-call assert p.returncode == 0 local_runs = local_instance.get_runs() assert len(local_runs) == 1 run_id = local_runs[0].run_id pipeline_run = local_instance.get_run_by_id(run_id) assert pipeline_run assert pipeline_run.status == PipelineRunStatus.SUCCESS
def default_app(): instance = DagsterInstance.get() process_context = get_workspace_process_context_from_kwargs( instance=instance, version=__version__, read_only=False, kwargs={}, ) return create_app(process_context, app_path_prefix="", debug=True)
def export_command(run_id, output_file): with DagsterInstance.get() as instance: run = instance.get_run_by_id(run_id) if run is None: raise click.UsageError( "Could not find run with run_id '{}'.\n{}".format( run_id, _recent_failed_runs_text(instance))) export_run(instance, run, output_file)
def default_app(debug=False): instance = DagsterInstance.get() process_context = get_workspace_process_context_from_kwargs( instance=instance, version=__version__, read_only=False, kwargs={}, ) return DagitWebserver(process_context, ).create_asgi_app(debug=debug)
def run_loop(self, daemon_uuid, daemon_shutdown_event): # Each loop runs in its own thread with its own instance with DagsterInstance.get() as instance: while not daemon_shutdown_event.is_set(): curr_time = pendulum.now("UTC") if (not self._last_iteration_time or (curr_time - self._last_iteration_time).total_seconds() >= self.interval_seconds): self._last_iteration_time = curr_time self._run_iteration(instance, curr_time, daemon_uuid) daemon_shutdown_event.wait(0.5)
def export_command(run_id, output_file): with DagsterInstance.get() as instance: run = instance.get_run_by_id(run_id) if run is None: raise click.UsageError( "Could not find run with run_id '{}'.\n{}".format( run_id, _recent_failed_runs_text(instance))) debug_payload = DebugRunPayload.build(instance, run) with GzipFile(output_file, "wb") as file: click.echo("Exporting run_id '{}' to gzip output file {}.".format( run_id, output_file)) debug_payload.write(file)
def run_command(interval_seconds, max_concurrent_runs): coordinator = QueuedRunCoordinatorDaemon( DagsterInstance.get(), max_concurrent_runs=max_concurrent_runs) click.echo("Starting run coordinator") coordinator.run(interval_seconds=interval_seconds)
def script_example(): with instance_for_test(): # start_script execute_pipeline(predict_color, instance=DagsterInstance.get())