def create_app_with_execution_handle(handle, instance, reloader=None): check.inst_param(handle, 'handle', ExecutionTargetHandle) check.inst_param(instance, 'instance', DagsterInstance) check.opt_inst_param(reloader, 'reloader', Reloader) execution_manager = get_execution_manager(instance) warn_if_compute_logs_disabled() print('Loading repository...') context = DagsterGraphQLInProcessRepositoryContext( handle=handle, instance=instance, execution_manager=execution_manager, reloader=reloader, version=__version__, ) # Automatically initialize scheduler everytime Dagit loads scheduler = instance.scheduler repository = context.get_repository() if repository.schedule_defs: if scheduler: handle = context.get_handle() python_path = sys.executable repository_path = handle.data.repository_yaml reconcile_scheduler_state(python_path, repository_path, repository=repository, instance=instance) else: warnings.warn(MISSING_SCHEDULER_WARNING) return instantiate_app_with_views(context)
def test_start_schedule_fails(): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) schedule_def = repository.get_schedule_def("no_config_pipeline_every_min_schedule") def raises(*args, **kwargs): raise Exception('Patch') instance._scheduler._start_cron_job = raises # pylint: disable=protected-access with pytest.raises(Exception, match='Patch'): instance.start_schedule(repository, "no_config_pipeline_every_min_schedule") schedule = instance.get_schedule_by_name(repository, schedule_def.name) assert schedule.status == ScheduleStatus.STOPPED
def test_invalid_config_schedule_error(self, graphql_context, snapshot): repository = graphql_context.legacy_get_repository_definition() instance = graphql_context.instance reconcile_scheduler_state("", "", repository, instance) result = execute_dagster_graphql_and_finish_runs( graphql_context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'invalid_config_schedule'}, ) assert (result.data['startScheduledExecution']['__typename'] == 'PipelineConfigValidationInvalid') # Check tick data and stats through gql result = execute_dagster_graphql_and_finish_runs( graphql_context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'invalid_config_schedule') assert schedule_result['stats']['ticksSucceeded'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'invalid_config_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SUCCESS
def test_stop_schedule_unsuccessful(restore_cron_tab, ): # pylint:disable=unused-argument,redefined-outer-name with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) def do_nothing(*_): pass instance._scheduler._end_cron_job = do_nothing # pylint: disable=protected-access instance.start_schedule(repository, "no_config_pipeline_every_min_schedule") # End schedule with pytest.raises( DagsterInvariantViolationError, match= "Attempted to remove cron job for schedule no_config_pipeline_every_min_schedule, but failed.", ): instance.stop_schedule(repository, "no_config_pipeline_every_min_schedule")
def test_start_schedule_manual_delete_debug( restore_cron_tab, snapshot # pylint:disable=unused-argument,redefined-outer-name ): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path="fake path", repository_path="", repository=repository, instance=instance, ) instance.start_schedule(repository, "no_config_pipeline_every_min_schedule") # Manually delete the schedule from the crontab instance.scheduler._end_cron_job( # pylint: disable=protected-access instance, repository, instance.get_schedule_by_name( repository, "no_config_pipeline_every_min_schedule"), ) # Check debug command snapshot.assert_match(instance.scheduler_debug_info())
def test_re_init(): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule schedule = instance.start_schedule(repository, "no_config_pipeline_every_min_schedule") # Re-initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Check schedules are saved to disk assert 'schedules' in os.listdir(tempdir) schedules = instance.all_schedules(repository) for schedule in schedules: assert "/bin/python" in schedule.python_path
def test_tick_skip(snapshot): with seven.TemporaryDirectory() as temp_dir: instance = get_instance(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) repository = context.get_repository() reconcile_scheduler_state("", "", repository, instance) execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'no_config_should_execute'}, ) # Check tick data and stats through gql result = execute_dagster_graphql(context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'no_config_should_execute') assert schedule_result['stats']['ticksSkipped'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository, 'no_config_should_execute') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SKIPPED
def create_app_with_reconstructable_repo(recon_repo, instance, reloader=None): check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) check.inst_param(instance, 'instance', DagsterInstance) check.opt_inst_param(reloader, 'reloader', Reloader) warn_if_compute_logs_disabled() print('Loading repository...') context = DagsterGraphQLContext( instance=instance, locations=[InProcessRepositoryLocation(recon_repo, reloader=reloader)], version=__version__, ) # Automatically initialize scheduler everytime Dagit loads scheduler = instance.scheduler repository = context.legacy_get_repository_definition() if repository.schedule_defs: if scheduler: python_path = sys.executable repository_path = context.legacy_location.get_reconstructable_repository( ).yaml_path reconcile_scheduler_state(python_path, repository_path, repository=repository, instance=instance) else: warnings.warn(MISSING_SCHEDULER_WARNING) return instantiate_app_with_views(context)
def test_should_execute_scheduler_error(self, graphql_context, snapshot): instance = graphql_context.instance repository = graphql_context.legacy_get_repository_definition() reconcile_scheduler_state("", "", repository, instance) execute_dagster_graphql_and_finish_runs( graphql_context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'should_execute_error_schedule'}, ) # Check tick data and stats through gql result = execute_dagster_graphql_and_finish_runs( graphql_context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'should_execute_error_schedule') assert schedule_result['stats']['ticksFailed'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'should_execute_error_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.FAILURE assert tick.error assert ( "Error occurred during the execution should_execute for schedule " "should_execute_error_schedule" in tick.error.message)
def test_tick_skip(self, graphql_context, snapshot): instance = graphql_context.instance repository = graphql_context.legacy_get_repository_definition() reconcile_scheduler_state("", "", repository, instance) execute_dagster_graphql_and_finish_runs( graphql_context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'no_config_should_execute'}, ) # Check tick data and stats through gql result = execute_dagster_graphql_and_finish_runs( graphql_context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'no_config_should_execute') assert schedule_result['stats']['ticksSkipped'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'no_config_should_execute') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SKIPPED
def test_environment_dict_scheduler_error(self, graphql_context, snapshot): instance = graphql_context.instance repository = graphql_context.legacy_get_repository_definition() reconcile_scheduler_state("", "", repository, instance) result = execute_dagster_graphql_and_finish_runs( graphql_context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'environment_dict_error_schedule'}, ) assert_start_scheduled_execution_success(result) run_id = result.data['startScheduledExecution']['run']['runId'] # Check tick data and stats through gql result = execute_dagster_graphql_and_finish_runs( graphql_context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'environment_dict_error_schedule') assert schedule_result['stats']['ticksSucceeded'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'environment_dict_error_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SUCCESS assert tick.run_id == run_id
def test_start_and_stop_schedule(): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) schedule_def = repository.get_schedule_def("no_config_pipeline_every_min_schedule") # Start schedule schedule = instance.start_schedule(repository, "no_config_pipeline_every_min_schedule") check.inst_param(schedule, 'schedule', Schedule) assert "/bin/python" in schedule.python_path assert 'schedules' in os.listdir(tempdir) assert "{}.{}.sh".format(repository.name, schedule_def.name) in os.listdir( os.path.join(tempdir, 'schedules', 'scripts') ) # End schedule instance.stop_schedule(repository, "no_config_pipeline_every_min_schedule") assert "{}.{}.sh".format(repository.name, schedule_def.name) not in os.listdir( os.path.join(tempdir, 'schedules', 'scripts') )
def test_enviornment_dict_scheduler_error_serialize_cauze(): with seven.TemporaryDirectory() as temp_dir: instance = get_instance(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) repository = context.get_repository_definition() reconcile_scheduler_state("", "", repository, instance) result = execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'environment_dict_error_schedule'}, ) assert result.data['startScheduledExecution'][ '__typename'] == 'StartPipelineRunSuccess' run_id = result.data['startScheduledExecution']['run']['runId'] ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'environment_dict_error_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SUCCESS assert tick.run_id == run_id
def test_enviornment_dict_scheduler_error(snapshot): with seven.TemporaryDirectory() as temp_dir: instance = get_instance(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) repository = context.get_repository_definition() reconcile_scheduler_state("", "", repository, instance) result = execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'environment_dict_error_schedule'}, ) assert result.data['startScheduledExecution'][ '__typename'] == 'StartPipelineRunSuccess' run_id = result.data['startScheduledExecution']['run']['runId'] # Check tick data and stats through gql result = execute_dagster_graphql(context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'environment_dict_error_schedule') assert schedule_result['stats']['ticksSucceeded'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'environment_dict_error_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SUCCESS assert tick.run_id == run_id
def test_should_execute_scheduler_error(snapshot): with seven.TemporaryDirectory() as temp_dir: instance = get_instance(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) repository = context.get_repository_definition() reconcile_scheduler_state("", "", repository, instance) execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'should_execute_error_schedule'}, ) # Check tick data and stats through gql result = execute_dagster_graphql(context, SCHEDULE_TICKS_QUERY) schedule_result = next( x for x in result.data['scheduler']['runningSchedules'] if x['scheduleDefinition']['name'] == 'should_execute_error_schedule') assert schedule_result['stats']['ticksFailed'] == 1 snapshot.assert_match(schedule_result) ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'should_execute_error_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.FAILURE assert tick.error assert ( "Error occurred during the execution should_execute for schedule " "should_execute_error_schedule" in tick.error.message)
def test_schedules(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) # We need to call up on the scheduler handle to persist # state about the schedules to disk before running them. # Note: This dependency will be removed soon. repository = context.get_repository() reconcile_scheduler_state( python_path=sys.executable, repository_path=file_relative_path(__file__, '../'), repository=repository, instance=instance, ) for schedule_name in [ 'many_events_every_min', 'pandas_hello_world_hourly', ]: result = execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_MUTATION, variables={'scheduleName': schedule_name}, ) assert not result.errors assert result.data assert result.data['startScheduledExecution'][ '__typename'] == 'StartPipelineRunSuccess'
def test_get_all_schedules(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(temp_dir), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesystemTestScheduler(temp_dir), run_launcher=SyncInMemoryRunLauncher(), ) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) # Initialize scheduler repository = context.legacy_get_repository_definition() reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule schedule = instance.start_schedule( repository.name, "no_config_pipeline_hourly_schedule") # Query Scheduler + all Schedules scheduler_result = execute_dagster_graphql(context, GET_SCHEDULES_QUERY) # These schedules are defined in dagster_graphql_tests/graphql/setup_scheduler.py # If you add a schedule there, be sure to update the number of schedules below assert scheduler_result.data assert scheduler_result.data['scheduler'] assert scheduler_result.data['scheduler']['runningSchedules'] assert len( scheduler_result.data['scheduler']['runningSchedules']) == 18 for schedule in scheduler_result.data['scheduler']['runningSchedules']: if schedule['scheduleDefinition'][ 'name'] == 'no_config_pipeline_hourly_schedule': assert schedule['status'] == 'RUNNING' if schedule['scheduleDefinition'][ 'name'] == 'environment_dict_error_schedule': assert schedule['scheduleDefinition']['runConfigYaml'] is None elif schedule['scheduleDefinition'][ 'name'] == 'invalid_config_schedule': assert (schedule['scheduleDefinition']['runConfigYaml'] == 'solids:\n takes_an_enum:\n config: invalid\n') else: assert (schedule['scheduleDefinition']['runConfigYaml'] == 'storage:\n filesystem: {}\n')
def test_start_schedule_manual_duplicate_schedules_add_debug( restore_cron_tab, snapshot # pylint:disable=unused-argument,redefined-outer-name ): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path="fake path", repository_path="", repository=repository, instance=instance, ) instance.start_schedule(repository.name, "no_config_pipeline_every_min_schedule") # Manually add extra cron tabs instance.scheduler._start_cron_job( # pylint: disable=protected-access instance, repository.name, instance.get_schedule_by_name( repository.name, "no_config_pipeline_every_min_schedule"), ) instance.scheduler._start_cron_job( # pylint: disable=protected-access instance, repository.name, instance.get_schedule_by_name( repository.name, "no_config_pipeline_every_min_schedule"), ) # Check debug command debug_info = instance.scheduler_debug_info() assert len(debug_info.errors) == 1 # Reconcile should fix error reconcile_scheduler_state( python_path="fake path", repository_path="", repository=repository, instance=instance, ) debug_info = instance.scheduler_debug_info() assert len(debug_info.errors) == 0
def test_stop_schedule_fails( restore_cron_tab, # pylint:disable=unused-argument,redefined-outer-name ): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) schedule_def = repository.get_schedule_def( "no_config_pipeline_every_min_schedule") def raises(*args, **kwargs): raise Exception('Patch') instance._scheduler._end_cron_job = raises # pylint: disable=protected-access schedule = instance.start_schedule( repository.name, "no_config_pipeline_every_min_schedule") check.inst_param(schedule, 'schedule', Schedule) assert "/bin/python" in schedule.python_path assert 'schedules' in os.listdir(tempdir) assert "{}.{}.sh".format(repository.name, schedule_def.name) in os.listdir( os.path.join(tempdir, 'schedules', 'scripts')) # End schedule with pytest.raises(Exception, match='Patch'): instance.stop_schedule(repository.name, "no_config_pipeline_every_min_schedule") schedule = instance.get_schedule_by_name(repository.name, schedule_def.name) assert schedule.status == ScheduleStatus.RUNNING
def test_script_execution(restore_cron_tab, unset_dagster_home): # pylint:disable=unused-argument,redefined-outer-name with TemporaryDirectory() as tempdir: os.environ["DAGSTER_HOME"] = tempdir config = { 'scheduler': { 'module': 'dagster_cron', 'class': 'SystemCronScheduler', 'config': {} }, # This needs to synchronously execute to completion when # the generated bash script is invoked 'run_launcher': { 'module': 'dagster.core.launcher.sync_in_memory_run_launcher', 'class': 'SyncInMemoryRunLauncher', }, } with open(os.path.join(tempdir, 'dagster.yaml'), 'w+') as f: f.write(yaml.dump(config)) instance = DagsterInstance.get() repository = define_repository() # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path=file_relative_path(__file__, './repository.yaml'), repository=repository, instance=instance, ) instance.start_schedule(repository.name, "no_config_pipeline_every_min_schedule") schedule_def = repository.get_schedule_def( "no_config_pipeline_every_min_schedule") script = instance.scheduler._get_bash_script_file_path( # pylint: disable=protected-access instance, repository.name, schedule_def) subprocess.check_output([script], shell=True, env={"DAGSTER_HOME": tempdir}) runs = instance.get_runs() assert len(runs) == 1 assert runs[0].status == PipelineRunStatus.SUCCESS
def execute_up_command(preview, cli_args, print_fn): handle = recon_repo_for_cli_args(cli_args) instance = DagsterInstance.get() check_handle_and_scheduler(handle, instance) repository = handle.get_definition() python_path = sys.executable repository_path = handle.yaml_path print_changes(repository, instance, print_fn, preview=preview) if preview: return try: reconcile_scheduler_state(python_path, repository_path, repository, instance=instance) except DagsterInvariantViolationError as ex: raise click.UsageError(ex)
def test_start_stop_schedule(): with seven.TemporaryDirectory() as temp_dir: instance = DagsterInstance( instance_type=InstanceType.EPHEMERAL, local_artifact_storage=LocalArtifactStorage(temp_dir), run_storage=InMemoryRunStorage(), event_storage=InMemoryEventLogStorage(), compute_log_manager=NoOpComputeLogManager(temp_dir), schedule_storage=SqliteScheduleStorage.from_local(temp_dir), scheduler=FilesystemTestScheduler(temp_dir), run_launcher=SyncInMemoryRunLauncher(), ) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) # Initialize scheduler repository = context.legacy_get_repository_definition() reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule start_result = execute_dagster_graphql( context, START_SCHEDULES_QUERY, variables={'scheduleName': 'no_config_pipeline_hourly_schedule'}, ) assert start_result.data['startSchedule']['schedule'][ 'status'] == 'RUNNING' # Stop schedule stop_result = execute_dagster_graphql( context, STOP_SCHEDULES_QUERY, variables={'scheduleName': 'no_config_pipeline_hourly_schedule'}, ) assert stop_result.data['stopRunningSchedule']['schedule'][ 'status'] == 'STOPPED'
def test_query_multiple_schedule_ticks(self, graphql_context, snapshot): instance = graphql_context.instance repository = graphql_context.legacy_get_repository_definition() reconcile_scheduler_state("", "", repository, instance) for scheduleName in [ 'no_config_pipeline_hourly_schedule', 'no_config_should_execute', 'environment_dict_error_schedule', ]: execute_dagster_graphql_and_finish_runs( graphql_context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': scheduleName}, ) result = execute_dagster_graphql_and_finish_runs( graphql_context, SCHEDULE_TICKS_QUERY) snapshot.assert_match(result.data['scheduler']['runningSchedules'])
def test_tick_success(self, graphql_context, snapshot): context = graphql_context instance = context.instance repository = context.legacy_get_repository_definition() reconcile_scheduler_state("", "", repository, instance) schedule_def = repository.get_schedule_def( "no_config_pipeline_hourly_schedule") start_time = time.time() execute_dagster_graphql_and_finish_runs( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': schedule_def.name}, ) # Check tick data and stats through gql result = execute_dagster_graphql_and_finish_runs( context, SCHEDULE_TICKS_QUERY) assert result.data schedule_result = next( schedule_result for schedule_result in result.data['scheduler']['runningSchedules'] if schedule_result['scheduleDefinition']['name'] == schedule_def.name) assert schedule_result assert schedule_result['stats']['ticksSucceeded'] == 1 snapshot.assert_match(schedule_result) # Check directly against the DB ticks = instance.get_schedule_ticks_by_schedule( repository.name, schedule_def.name) assert len(ticks) == 1 tick = ticks[0] assert tick.schedule_name == schedule_def.name assert tick.cron_schedule == schedule_def.cron_schedule assert tick.timestamp > start_time and tick.timestamp < time.time() assert tick.status == ScheduleTickStatus.SUCCESS assert tick.run_id
def test_tick_success(snapshot): with seven.TemporaryDirectory() as temp_dir: instance = get_instance(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) repository = context.get_repository_definition() reconcile_scheduler_state("", "", repository, instance) schedule_def = repository.get_schedule_def( "no_config_pipeline_hourly_schedule") start_time = time.time() execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': schedule_def.name}, ) # Check tick data and stats through gql result = execute_dagster_graphql(context, SCHEDULE_TICKS_QUERY) schedule_result = next( schedule_result for schedule_result in result.data['scheduler']['runningSchedules'] if schedule_result['scheduleDefinition']['name'] == schedule_def.name) assert schedule_result assert schedule_result['stats']['ticksSucceeded'] == 1 snapshot.assert_match(schedule_result) # Check directly against the DB ticks = instance.get_schedule_ticks_by_schedule( repository.name, schedule_def.name) assert len(ticks) == 1 tick = ticks[0] assert tick.schedule_name == schedule_def.name assert tick.cron_schedule == schedule_def.cron_schedule assert tick.timestamp > start_time and tick.timestamp < time.time() assert tick.status == ScheduleTickStatus.SUCCESS assert tick.run_id
def test_init(restore_cron_tab): # pylint:disable=unused-argument,redefined-outer-name with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Check schedules are saved to disk assert 'schedules' in os.listdir(tempdir) schedules = instance.all_schedules(repository) for schedule in schedules: assert "/bin/python" in schedule.python_path
def test_environment_dict_scheduler_error_serialize_cause( self, graphql_context): instance = graphql_context.instance repository = graphql_context.legacy_get_repository_definition() reconcile_scheduler_state("", "", repository, instance) result = execute_dagster_graphql_and_finish_runs( graphql_context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': 'environment_dict_error_schedule'}, ) assert_start_scheduled_execution_success(result) run_id = result.data['startScheduledExecution']['run']['runId'] ticks = instance.get_schedule_ticks_by_schedule( repository.name, 'environment_dict_error_schedule') assert len(ticks) == 1 tick = ticks[0] assert tick.status == ScheduleTickStatus.SUCCESS assert tick.run_id == run_id
def test_wipe(): with TemporaryDirectory() as tempdir: repository = RepositoryDefinition(name="test_repository", schedule_defs=define_schedules()) instance = define_scheduler_instance(tempdir) # Initialize scheduler reconcile_scheduler_state( python_path=sys.executable, repository_path="", repository=repository, instance=instance, ) # Start schedule instance.start_schedule(repository, "no_config_pipeline_every_min_schedule") # Wipe scheduler instance.wipe_all_schedules() # Check schedules are wiped assert instance.all_schedules(repository) == []
def test_query_multiple_schedule_ticks(snapshot): with seven.TemporaryDirectory() as temp_dir: instance = get_instance(temp_dir) context = define_context_for_repository_yaml(path=file_relative_path( __file__, '../repository.yaml'), instance=instance) repository = context.get_repository_definition() reconcile_scheduler_state("", "", repository, instance) for scheduleName in [ 'no_config_pipeline_hourly_schedule', 'no_config_should_execute', 'environment_dict_error_schedule', ]: execute_dagster_graphql( context, START_SCHEDULED_EXECUTION_QUERY, variables={'scheduleName': scheduleName}, ) result = execute_dagster_graphql(context, SCHEDULE_TICKS_QUERY) snapshot.assert_match(result.data['scheduler']['runningSchedules'])