def dagster_instance(helm_postgres_url, run_launcher): # pylint: disable=redefined-outer-name
tempdir = DagsterInstance.temp_storage()
with DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(helm_postgres_url),
event_storage=PostgresEventLogStorage(helm_postgres_url),
compute_log_manager=NoOpComputeLogManager(),
run_coordinator=DefaultRunCoordinator(),
run_launcher=run_launcher,
) as instance:
yield instance
def dagster_instance(run_launcher, network_postgres):
tempdir = DagsterInstance.temp_storage()
postgres_url = 'postgresql://*****:*****@localhost:{network_postgres}/test'.format(
network_postgres=network_postgres)
instance = DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(postgres_url),
event_storage=PostgresEventLogStorage(postgres_url),
compute_log_manager=NoOpComputeLogManager(
compute_logs_directory(tempdir)),
run_launcher=run_launcher,
)
return instance
def dagster_instance(helm_namespace, run_launcher): # pylint: disable=redefined-outer-name
tempdir = DagsterInstance.temp_storage()
with local_port_forward_postgres(
namespace=helm_namespace) as local_forward_port:
postgres_url = 'postgresql://*****:*****@localhost:{local_forward_port}/test'.format(
local_forward_port=local_forward_port)
print('Local Postgres forwarding URL: ', postgres_url)
instance = DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(postgres_url),
event_storage=PostgresEventLogStorage(postgres_url),
compute_log_manager=NoOpComputeLogManager(),
run_launcher=run_launcher,
)
yield instance
def dagster_instance_for_daemon(
helm_postgres_url_for_daemon,
): # pylint: disable=redefined-outer-name
tempdir = DagsterInstance.temp_storage()
with DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(helm_postgres_url_for_daemon),
event_storage=PostgresEventLogStorage(helm_postgres_url_for_daemon),
schedule_storage=PostgresScheduleStorage(helm_postgres_url_for_daemon),
compute_log_manager=NoOpComputeLogManager(),
run_coordinator=QueuedRunCoordinator(),
run_launcher=ExplodingRunLauncher(),
scheduler=DagsterDaemonScheduler(),
) as instance:
yield instance
check_export_runs(instance)
def dagster_instance_for_user_deployments_subchart_disabled(
helm_postgres_url_for_user_deployments_subchart_disabled,
): # pylint: disable=redefined-outer-name
tempdir = DagsterInstance.temp_storage()
with DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(helm_postgres_url_for_user_deployments_subchart_disabled),
event_storage=PostgresEventLogStorage(
helm_postgres_url_for_user_deployments_subchart_disabled
),
compute_log_manager=NoOpComputeLogManager(),
run_coordinator=DefaultRunCoordinator(),
run_launcher=ExplodingRunLauncher(),
) as instance:
yield instance
check_export_runs(instance)
def dagster_instance_for_k8s_run_launcher(
helm_postgres_url_for_k8s_run_launcher,
): # pylint: disable=redefined-outer-name
tempdir = DagsterInstance.temp_storage()
instance_ref = InstanceRef.from_dir(tempdir)
with DagsterInstance(
instance_type=InstanceType.PERSISTENT,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(helm_postgres_url_for_k8s_run_launcher),
event_storage=PostgresEventLogStorage(helm_postgres_url_for_k8s_run_launcher),
schedule_storage=PostgresScheduleStorage(helm_postgres_url_for_k8s_run_launcher),
compute_log_manager=NoOpComputeLogManager(),
run_coordinator=DefaultRunCoordinator(),
run_launcher=ExplodingRunLauncher(),
ref=instance_ref,
) as instance:
yield instance
check_export_runs(instance)
def dagster_instance_for_daemon(helm_namespace_for_daemon, run_launcher): # pylint: disable=redefined-outer-name
tempdir = DagsterInstance.temp_storage()
with local_port_forward_postgres(
namespace=helm_namespace_for_daemon) as local_forward_port:
postgres_url = "postgresql://*****:*****@localhost:{local_forward_port}/test".format(
local_forward_port=local_forward_port)
print("Local Postgres forwarding URL: ", postgres_url)
instance = DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(postgres_url),
event_storage=PostgresEventLogStorage(postgres_url),
schedule_storage=PostgresScheduleStorage(postgres_url),
compute_log_manager=NoOpComputeLogManager(),
run_coordinator=QueuedRunCoordinator(),
run_launcher=run_launcher,
scheduler=DagsterDaemonScheduler(),
)
yield instance
def dagster_instance_for_k8s_run_launcher(helm_namespace_for_k8s_run_launcher, run_launcher):
tempdir = DagsterInstance.temp_storage()
with local_port_forward_postgres(
namespace=helm_namespace_for_k8s_run_launcher
) as local_forward_port:
postgres_url = "postgresql://*****:*****@localhost:{local_forward_port}/test".format(
local_forward_port=local_forward_port
)
print("Local Postgres forwarding URL: ", postgres_url)
instance = DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(postgres_url),
event_storage=PostgresEventLogStorage(postgres_url),
schedule_storage=PostgresScheduleStorage(postgres_url),
compute_log_manager=NoOpComputeLogManager(),
run_launcher=run_launcher,
)
yield instance
def dagster_instance(helm_namespace, run_launcher): # pylint: disable=redefined-outer-name
@contextmanager
def local_port_forward_postgres():
print('Port-forwarding postgres')
postgres_pod_name = (check_output([
'kubectl',
'get',
'pods',
'--namespace',
helm_namespace,
'-l',
'app=postgresql,release=dagster',
'-o',
'jsonpath="{.items[0].metadata.name}"',
]).decode('utf-8').strip('"'))
forward_port = find_free_port()
wait_for_pod(postgres_pod_name, namespace=helm_namespace)
try:
p = subprocess.Popen([
'kubectl',
'port-forward',
'--namespace',
helm_namespace,
postgres_pod_name,
'{forward_port}:5432'.format(forward_port=forward_port),
])
# Validate port forwarding works
start = time.time()
while True:
if time.time() - start > PG_PORT_FORWARDING_TIMEOUT:
raise Exception(
'Timed out while waiting for postgres port forwarding')
print(
'Waiting for port forwarding from k8s pod %s:5432 to localhost:%d to be'
' available...' % (postgres_pod_name, forward_port))
try:
conn = psycopg2.connect(
database='test',
user='******',
password='******',
host='localhost',
port=forward_port,
)
conn.close()
break
except: # pylint: disable=bare-except, broad-except
time.sleep(1)
continue
yield forward_port
finally:
print('Terminating port-forwarding')
p.terminate()
tempdir = DagsterInstance.temp_storage()
with local_port_forward_postgres() as local_forward_port:
postgres_url = 'postgresql://*****:*****@localhost:{local_forward_port}/test'.format(
local_forward_port=local_forward_port)
print('Local Postgres forwarding URL: ', postgres_url)
instance = DagsterInstance(
instance_type=InstanceType.EPHEMERAL,
local_artifact_storage=LocalArtifactStorage(tempdir),
run_storage=PostgresRunStorage(postgres_url),
event_storage=PostgresEventLogStorage(postgres_url),
compute_log_manager=NoOpComputeLogManager(),
run_launcher=run_launcher,
)
yield instance