def dagster_instance(helm_postgres_url, run_launcher): # pylint: disable=redefined-outer-name tempdir = DagsterInstance.temp_storage() with DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(helm_postgres_url), event_storage=PostgresEventLogStorage(helm_postgres_url), compute_log_manager=NoOpComputeLogManager(), run_coordinator=DefaultRunCoordinator(), run_launcher=run_launcher, ) as instance: yield instance
def dagster_instance(run_launcher, network_postgres): tempdir = DagsterInstance.temp_storage() postgres_url = 'postgresql://*****:*****@localhost:{network_postgres}/test'.format( network_postgres=network_postgres) instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(postgres_url), event_storage=PostgresEventLogStorage(postgres_url), compute_log_manager=NoOpComputeLogManager( compute_logs_directory(tempdir)), run_launcher=run_launcher, ) return instance
def dagster_instance(helm_postgres_url): # pylint: disable=redefined-outer-name with tempfile.TemporaryDirectory() as tempdir: with environ({"DAGSTER_HOME": tempdir}): with DagsterInstance( instance_type=InstanceType.PERSISTENT, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(helm_postgres_url), event_storage=PostgresEventLogStorage(helm_postgres_url), compute_log_manager=NoOpComputeLogManager(), run_coordinator=DefaultRunCoordinator(), run_launcher=ExplodingRunLauncher(), # use graphql to launch any runs ref=InstanceRef.from_dir(tempdir), ) as instance: yield instance check_export_runs(instance)
def dagster_instance(helm_namespace, run_launcher): # pylint: disable=redefined-outer-name tempdir = DagsterInstance.temp_storage() with local_port_forward_postgres( namespace=helm_namespace) as local_forward_port: postgres_url = 'postgresql://*****:*****@localhost:{local_forward_port}/test'.format( local_forward_port=local_forward_port) print('Local Postgres forwarding URL: ', postgres_url) instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(postgres_url), event_storage=PostgresEventLogStorage(postgres_url), compute_log_manager=NoOpComputeLogManager(), run_launcher=run_launcher, ) yield instance
def dagster_instance_for_daemon( helm_postgres_url_for_daemon, ): # pylint: disable=redefined-outer-name tempdir = DagsterInstance.temp_storage() with DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(helm_postgres_url_for_daemon), event_storage=PostgresEventLogStorage(helm_postgres_url_for_daemon), schedule_storage=PostgresScheduleStorage(helm_postgres_url_for_daemon), compute_log_manager=NoOpComputeLogManager(), run_coordinator=QueuedRunCoordinator(), run_launcher=ExplodingRunLauncher(), scheduler=DagsterDaemonScheduler(), ) as instance: yield instance check_export_runs(instance)
def dagster_instance_for_user_deployments_subchart_disabled( helm_postgres_url_for_user_deployments_subchart_disabled, ): # pylint: disable=redefined-outer-name tempdir = DagsterInstance.temp_storage() with DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(helm_postgres_url_for_user_deployments_subchart_disabled), event_storage=PostgresEventLogStorage( helm_postgres_url_for_user_deployments_subchart_disabled ), compute_log_manager=NoOpComputeLogManager(), run_coordinator=DefaultRunCoordinator(), run_launcher=ExplodingRunLauncher(), ) as instance: yield instance check_export_runs(instance)
def dagster_instance_for_k8s_run_launcher( helm_postgres_url_for_k8s_run_launcher, ): # pylint: disable=redefined-outer-name tempdir = DagsterInstance.temp_storage() instance_ref = InstanceRef.from_dir(tempdir) with DagsterInstance( instance_type=InstanceType.PERSISTENT, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(helm_postgres_url_for_k8s_run_launcher), event_storage=PostgresEventLogStorage(helm_postgres_url_for_k8s_run_launcher), schedule_storage=PostgresScheduleStorage(helm_postgres_url_for_k8s_run_launcher), compute_log_manager=NoOpComputeLogManager(), run_coordinator=DefaultRunCoordinator(), run_launcher=ExplodingRunLauncher(), ref=instance_ref, ) as instance: yield instance check_export_runs(instance)
def dagster_instance_for_daemon(helm_namespace_for_daemon, run_launcher): # pylint: disable=redefined-outer-name tempdir = DagsterInstance.temp_storage() with local_port_forward_postgres( namespace=helm_namespace_for_daemon) as local_forward_port: postgres_url = "postgresql://*****:*****@localhost:{local_forward_port}/test".format( local_forward_port=local_forward_port) print("Local Postgres forwarding URL: ", postgres_url) instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(postgres_url), event_storage=PostgresEventLogStorage(postgres_url), schedule_storage=PostgresScheduleStorage(postgres_url), compute_log_manager=NoOpComputeLogManager(), run_coordinator=QueuedRunCoordinator(), run_launcher=run_launcher, scheduler=DagsterDaemonScheduler(), ) yield instance
def dagster_instance_for_k8s_run_launcher(helm_namespace_for_k8s_run_launcher, run_launcher): tempdir = DagsterInstance.temp_storage() with local_port_forward_postgres( namespace=helm_namespace_for_k8s_run_launcher ) as local_forward_port: postgres_url = "postgresql://*****:*****@localhost:{local_forward_port}/test".format( local_forward_port=local_forward_port ) print("Local Postgres forwarding URL: ", postgres_url) instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(postgres_url), event_storage=PostgresEventLogStorage(postgres_url), schedule_storage=PostgresScheduleStorage(postgres_url), compute_log_manager=NoOpComputeLogManager(), run_launcher=run_launcher, ) yield instance
def dagster_instance(helm_namespace, run_launcher): # pylint: disable=redefined-outer-name @contextmanager def local_port_forward_postgres(): print('Port-forwarding postgres') postgres_pod_name = (check_output([ 'kubectl', 'get', 'pods', '--namespace', helm_namespace, '-l', 'app=postgresql,release=dagster', '-o', 'jsonpath="{.items[0].metadata.name}"', ]).decode('utf-8').strip('"')) forward_port = find_free_port() wait_for_pod(postgres_pod_name, namespace=helm_namespace) try: p = subprocess.Popen([ 'kubectl', 'port-forward', '--namespace', helm_namespace, postgres_pod_name, '{forward_port}:5432'.format(forward_port=forward_port), ]) # Validate port forwarding works start = time.time() while True: if time.time() - start > PG_PORT_FORWARDING_TIMEOUT: raise Exception( 'Timed out while waiting for postgres port forwarding') print( 'Waiting for port forwarding from k8s pod %s:5432 to localhost:%d to be' ' available...' % (postgres_pod_name, forward_port)) try: conn = psycopg2.connect( database='test', user='******', password='******', host='localhost', port=forward_port, ) conn.close() break except: # pylint: disable=bare-except, broad-except time.sleep(1) continue yield forward_port finally: print('Terminating port-forwarding') p.terminate() tempdir = DagsterInstance.temp_storage() with local_port_forward_postgres() as local_forward_port: postgres_url = 'postgresql://*****:*****@localhost:{local_forward_port}/test'.format( local_forward_port=local_forward_port) print('Local Postgres forwarding URL: ', postgres_url) instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=PostgresRunStorage(postgres_url), event_storage=PostgresEventLogStorage(postgres_url), compute_log_manager=NoOpComputeLogManager(), run_launcher=run_launcher, ) yield instance