def grpc_command( port=None, socket=None, host="localhost", max_workers=None, heartbeat=False, heartbeat_timeout=30, lazy_load_user_code=False, ipc_output_file=None, fixed_server_id=None, override_system_timezone=None, **kwargs, ): if seven.IS_WINDOWS and port is None: raise click.UsageError( "You must pass a valid --port/-p on Windows: --socket/-s not supported." ) if not (port or socket and not (port and socket)): raise click.UsageError("You must pass one and only one of --port/-p or --socket/-s.") loadable_target_origin = None if any( kwargs[key] for key in [ "attribute", "working_directory", "module_name", "package_name", "python_file", "empty_working_directory", ] ): loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, attribute=kwargs["attribute"], working_directory=get_working_directory_from_kwargs(kwargs), module_name=kwargs["module_name"], python_file=kwargs["python_file"], package_name=kwargs["package_name"], ) with ( mock_system_timezone(override_system_timezone) if override_system_timezone else nullcontext() ): server = DagsterGrpcServer( port=port, socket=socket, host=host, loadable_target_origin=loadable_target_origin, max_workers=max_workers, heartbeat=heartbeat, heartbeat_timeout=heartbeat_timeout, lazy_load_user_code=lazy_load_user_code, ipc_output_file=ipc_output_file, fixed_server_id=fixed_server_id, ) server.serve()
def test_script_execution(restore_cron_tab, unset_dagster_home): # pylint:disable=unused-argument,redefined-outer-name with TemporaryDirectory() as tempdir: os.environ["DAGSTER_HOME"] = tempdir config = { "scheduler": { "module": "dagster_cron", "class": "SystemCronScheduler", "config": {} }, # This needs to synchronously execute to completion when # the generated bash script is invoked "run_launcher": { "module": "dagster.core.launcher.sync_in_memory_run_launcher", "class": "SyncInMemoryRunLauncher", }, } with open(os.path.join(tempdir, "dagster.yaml"), "w+") as f: f.write(yaml.dump(config)) instance = DagsterInstance.get() with mock_system_timezone("US/Eastern"): with get_test_external_repo() as external_repo: instance.start_schedule_and_update_storage_state( external_repo.get_external_schedule( "no_config_pipeline_every_min_schedule")) schedule_origin_id = external_repo.get_external_schedule( "no_config_pipeline_every_min_schedule" ).get_external_origin_id() script = instance.scheduler._get_bash_script_file_path( # pylint: disable=protected-access instance, schedule_origin_id) subprocess.check_output( [script], shell=True, env={"DAGSTER_HOME": tempdir}, ) runs = instance.get_runs() assert len(runs) == 1 assert runs[0].status == PipelineRunStatus.SUCCESS
def launch_scheduled_execution(output_file, schedule_name, override_system_timezone, **kwargs): with (mock_system_timezone(override_system_timezone) if override_system_timezone else nullcontext()): with ipc_write_stream(output_file) as stream: with DagsterInstance.get() as instance: repository_origin = get_repository_origin_from_kwargs(kwargs) job_origin = repository_origin.get_job_origin(schedule_name) # open the tick scope before we load any external artifacts so that # load errors are stored in DB with _schedule_tick_context( instance, stream, JobTickData( job_origin_id=job_origin.get_id(), job_name=schedule_name, job_type=JobType.SCHEDULE, status=JobTickStatus.STARTED, timestamp=time.time(), ), ) as tick_context: with get_repository_location_from_kwargs( kwargs) as repo_location: repo_dict = repo_location.get_repositories() check.invariant( repo_dict and len(repo_dict) == 1, "Passed in arguments should reference exactly one repository, instead there are {num_repos}" .format(num_repos=len(repo_dict)), ) external_repo = next(iter(repo_dict.values())) if not schedule_name in [ schedule.name for schedule in external_repo.get_external_schedules() ]: raise DagsterInvariantViolationError( "Could not find schedule named {schedule_name}" .format(schedule_name=schedule_name), ) external_schedule = external_repo.get_external_schedule( schedule_name) # Validate that either the schedule has no timezone or it matches # the system timezone schedule_timezone = external_schedule.execution_timezone if schedule_timezone: system_timezone = pendulum.now().timezone.name if system_timezone != external_schedule.execution_timezone: raise DagsterInvariantViolationError( "Schedule {schedule_name} is set to execute in {schedule_timezone}, " "but this scheduler can only run in the system timezone, " "{system_timezone}. Use DagsterDaemonScheduler if you want to be able " "to execute schedules in arbitrary timezones." .format( schedule_name=external_schedule.name, schedule_timezone=schedule_timezone, system_timezone=system_timezone, ), ) _launch_scheduled_executions(instance, repo_location, external_repo, external_schedule, tick_context)
def grpc_command( port=None, socket=None, host=None, max_workers=None, heartbeat=False, heartbeat_timeout=30, lazy_load_user_code=False, ipc_output_file=None, fixed_server_id=None, override_system_timezone=None, log_level="INFO", use_python_environment_entry_point=False, container_context=None, **kwargs, ): if seven.IS_WINDOWS and port is None: raise click.UsageError( "You must pass a valid --port/-p on Windows: --socket/-s not supported." ) if not (port or socket and not (port and socket)): raise click.UsageError( "You must pass one and only one of --port/-p or --socket/-s.") configure_loggers(log_level=coerce_valid_log_level(log_level)) logger = logging.getLogger("dagster.code_server") loadable_target_origin = None if any(kwargs[key] for key in [ "attribute", "working_directory", "module_name", "package_name", "python_file", "empty_working_directory", ]): loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, attribute=kwargs["attribute"], working_directory=(None if kwargs.get("empty_working_directory") else get_working_directory_from_kwargs(kwargs)), module_name=kwargs["module_name"], python_file=kwargs["python_file"], package_name=kwargs["package_name"], ) with (mock_system_timezone(override_system_timezone) if override_system_timezone else nullcontext()): server = DagsterGrpcServer( port=port, socket=socket, host=host, loadable_target_origin=loadable_target_origin, max_workers=max_workers, heartbeat=heartbeat, heartbeat_timeout=heartbeat_timeout, lazy_load_user_code=lazy_load_user_code, ipc_output_file=ipc_output_file, fixed_server_id=fixed_server_id, entry_point=(get_python_environment_entry_point(sys.executable) if use_python_environment_entry_point else DEFAULT_DAGSTER_ENTRY_POINT), container_context=json.loads(container_context) if container_context != None else None, ) code_desc = " " if loadable_target_origin: if loadable_target_origin.python_file: code_desc = f" for file {loadable_target_origin.python_file} " elif loadable_target_origin.package_name: code_desc = f" for package {loadable_target_origin.package_name} " elif loadable_target_origin.module_name: code_desc = f" for module {loadable_target_origin.module_name} " server_desc = ( f"Dagster code server{code_desc}on port {port} in process {os.getpid()}" if port else f"Dagster code server{code_desc}in process {os.getpid()}") logger.info("Started {server_desc}".format(server_desc=server_desc)) try: server.serve() finally: logger.info( "Shutting down {server_desc}".format(server_desc=server_desc))