def dagster_instance_with_k8s_scheduler( helm_namespace_for_k8s_run_launcher, run_launcher, k8s_scheduler, schedule_tempdir ): with local_port_forward_postgres( namespace=helm_namespace_for_k8s_run_launcher ) as local_forward_port: postgres_url = "postgresql://*****:*****@localhost:{local_forward_port}/test".format( local_forward_port=local_forward_port ) print("Local Postgres forwarding URL: ", postgres_url) instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(schedule_tempdir), run_storage=SqliteRunStorage.from_local(os.path.join(schedule_tempdir, "runs")), event_storage=PostgresEventLogStorage(postgres_url), compute_log_manager=NoOpComputeLogManager(), run_coordinator=DefaultRunCoordinator(), run_launcher=run_launcher, schedule_storage=SqliteScheduleStorage.from_local( os.path.join(schedule_tempdir, "schedules") ), scheduler=k8s_scheduler, ) yield instance
def test_get_schedule(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesystemTestScheduler(temp_dir), run_launcher=SyncInMemoryRunLauncher(), ) context = define_test_context(instance) # Initialize scheduler repository = context.legacy_get_repository_definition() instance.reconcile_scheduler_state( repository=repository, python_path='/path/to/python', repository_path='/path/to/repository', ) result = execute_dagster_graphql( context, GET_SCHEDULE, variables={'scheduleName': 'partition_based_multi_mode_decorator'}, ) assert result.data assert result.data['scheduleOrError']['__typename'] == 'RunningSchedule' assert result.data['scheduleOrError']['scheduleDefinition']['partitionSet']
def test_get_all_schedules(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(temp_dir), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesystemTestScheduler(temp_dir), run_launcher=SyncInMemoryRunLauncher(), ) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) # Initialize scheduler repository = context.legacy_get_repository_definition() reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule schedule = instance.start_schedule( repository.name, "no_config_pipeline_hourly_schedule") # Query Scheduler + all Schedules scheduler_result = execute_dagster_graphql(context, GET_SCHEDULES_QUERY) # These schedules are defined in dagster_graphql_tests/graphql/setup_scheduler.py # If you add a schedule there, be sure to update the number of schedules below assert scheduler_result.data assert scheduler_result.data['scheduler'] assert scheduler_result.data['scheduler']['runningSchedules'] assert len( scheduler_result.data['scheduler']['runningSchedules']) == 18 for schedule in scheduler_result.data['scheduler']['runningSchedules']: if schedule['scheduleDefinition'][ 'name'] == 'no_config_pipeline_hourly_schedule': assert schedule['status'] == 'RUNNING' if schedule['scheduleDefinition'][ 'name'] == 'environment_dict_error_schedule': assert schedule['scheduleDefinition']['runConfigYaml'] is None elif schedule['scheduleDefinition'][ 'name'] == 'invalid_config_schedule': assert (schedule['scheduleDefinition']['runConfigYaml'] == 'solids:\n takes_an_enum:\n config: invalid\n') else: assert (schedule['scheduleDefinition']['runConfigYaml'] == 'storage:\n filesystem: {}\n')
def define_scheduler_instance(tempdir): return DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(tempdir), schedule_storage=SqliteScheduleStorage.from_local(os.path.join(tempdir, 'schedules')), scheduler=SystemCronScheduler(os.path.join(tempdir, 'schedules')), )
def define_scheduler_instance(): with seven.TemporaryDirectory() as temp_dir: yield DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesytemTestScheduler(temp_dir), compute_log_manager=NoOpComputeLogManager(temp_dir), )
def define_scheduler_instance(tempdir): with pytest.warns(UserWarning, match="`SystemCronScheduler` is deprecated"): return DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(tempdir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(), schedule_storage=SqliteScheduleStorage.from_local(os.path.join(tempdir, "schedules")), scheduler=SystemCronScheduler(), run_coordinator=DefaultRunCoordinator(), run_launcher=SyncInMemoryRunLauncher(), )
def dagster_instance_with_k8s_scheduler(helm_postgres_url_for_k8s_run_launcher, run_launcher, k8s_scheduler, schedule_tempdir): # pylint: disable=redefined-outer-name with DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(schedule_tempdir), run_storage=SqliteRunStorage.from_local( os.path.join(schedule_tempdir, "runs")), event_storage=PostgresEventLogStorage( helm_postgres_url_for_k8s_run_launcher), compute_log_manager=NoOpComputeLogManager(), run_coordinator=DefaultRunCoordinator(), run_launcher=run_launcher, schedule_storage=SqliteScheduleStorage.from_local( os.path.join(schedule_tempdir, "schedules")), scheduler=k8s_scheduler, ) as instance: yield instance
def test_start_stop_schedule(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(temp_dir), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesystemTestScheduler(temp_dir), run_launcher=SyncInMemoryRunLauncher(), ) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) # Initialize scheduler repository = context.legacy_get_repository_definition() reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule start_result = execute_dagster_graphql( context, START_SCHEDULES_QUERY, variables={'scheduleName': 'no_config_pipeline_hourly_schedule'}, ) assert start_result.data['startSchedule']['schedule'][ 'status'] == 'RUNNING' # Stop schedule stop_result = execute_dagster_graphql( context, STOP_SCHEDULES_QUERY, variables={'scheduleName': 'no_config_pipeline_hourly_schedule'}, ) assert stop_result.data['stopRunningSchedule']['schedule'][ 'status'] == 'STOPPED'
def test_get_all_schedules(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(temp_dir), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesytemTestScheduler(temp_dir), ) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) # Initialize scheduler repository = context.get_repository() scheduler_handle = context.scheduler_handle scheduler_handle.up( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule schedule = instance.start_schedule( repository, "no_config_pipeline_hourly_schedule") # Query Scheduler + all Schedules scheduler_result = execute_dagster_graphql(context, GET_SCHEDULES_QUERY) assert scheduler_result.data assert scheduler_result.data['scheduler'] assert scheduler_result.data['scheduler']['runningSchedules'] assert len( scheduler_result.data['scheduler']['runningSchedules']) == 11 for schedule in scheduler_result.data['scheduler']['runningSchedules']: assert (schedule['scheduleDefinition']['environmentConfigYaml'] == 'storage:\n filesystem: {}\n')
def create_sqlite_schedule_storage(): with tempfile.TemporaryDirectory() as tempdir: yield SqliteScheduleStorage.from_local(tempdir)