def grpc_command(
port=None,
socket=None,
host="localhost",
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
ipc_output_file=None,
fixed_server_id=None,
override_system_timezone=None,
**kwargs,
):
if seven.IS_WINDOWS and port is None:
raise click.UsageError(
"You must pass a valid --port/-p on Windows: --socket/-s not supported."
)
if not (port or socket and not (port and socket)):
raise click.UsageError("You must pass one and only one of --port/-p or --socket/-s.")
loadable_target_origin = None
if any(
kwargs[key]
for key in [
"attribute",
"working_directory",
"module_name",
"package_name",
"python_file",
"empty_working_directory",
]
):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
attribute=kwargs["attribute"],
working_directory=get_working_directory_from_kwargs(kwargs),
module_name=kwargs["module_name"],
python_file=kwargs["python_file"],
package_name=kwargs["package_name"],
)
with (
mock_system_timezone(override_system_timezone)
if override_system_timezone
else nullcontext()
):
server = DagsterGrpcServer(
port=port,
socket=socket,
host=host,
loadable_target_origin=loadable_target_origin,
max_workers=max_workers,
heartbeat=heartbeat,
heartbeat_timeout=heartbeat_timeout,
lazy_load_user_code=lazy_load_user_code,
ipc_output_file=ipc_output_file,
fixed_server_id=fixed_server_id,
)
server.serve()
def test_script_execution(restore_cron_tab, unset_dagster_home): # pylint:disable=unused-argument,redefined-outer-name
with TemporaryDirectory() as tempdir:
os.environ["DAGSTER_HOME"] = tempdir
config = {
"scheduler": {
"module": "dagster_cron",
"class": "SystemCronScheduler",
"config": {}
},
# This needs to synchronously execute to completion when
# the generated bash script is invoked
"run_launcher": {
"module": "dagster.core.launcher.sync_in_memory_run_launcher",
"class": "SyncInMemoryRunLauncher",
},
}
with open(os.path.join(tempdir, "dagster.yaml"), "w+") as f:
f.write(yaml.dump(config))
instance = DagsterInstance.get()
with mock_system_timezone("US/Eastern"):
with get_test_external_repo() as external_repo:
instance.start_schedule_and_update_storage_state(
external_repo.get_external_schedule(
"no_config_pipeline_every_min_schedule"))
schedule_origin_id = external_repo.get_external_schedule(
"no_config_pipeline_every_min_schedule"
).get_external_origin_id()
script = instance.scheduler._get_bash_script_file_path( # pylint: disable=protected-access
instance, schedule_origin_id)
subprocess.check_output(
[script],
shell=True,
env={"DAGSTER_HOME": tempdir},
)
runs = instance.get_runs()
assert len(runs) == 1
assert runs[0].status == PipelineRunStatus.SUCCESS
def launch_scheduled_execution(output_file, schedule_name,
override_system_timezone, **kwargs):
with (mock_system_timezone(override_system_timezone)
if override_system_timezone else nullcontext()):
with ipc_write_stream(output_file) as stream:
with DagsterInstance.get() as instance:
repository_origin = get_repository_origin_from_kwargs(kwargs)
job_origin = repository_origin.get_job_origin(schedule_name)
# open the tick scope before we load any external artifacts so that
# load errors are stored in DB
with _schedule_tick_context(
instance,
stream,
JobTickData(
job_origin_id=job_origin.get_id(),
job_name=schedule_name,
job_type=JobType.SCHEDULE,
status=JobTickStatus.STARTED,
timestamp=time.time(),
),
) as tick_context:
with get_repository_location_from_kwargs(
kwargs) as repo_location:
repo_dict = repo_location.get_repositories()
check.invariant(
repo_dict and len(repo_dict) == 1,
"Passed in arguments should reference exactly one repository, instead there are {num_repos}"
.format(num_repos=len(repo_dict)),
)
external_repo = next(iter(repo_dict.values()))
if not schedule_name in [
schedule.name for schedule in
external_repo.get_external_schedules()
]:
raise DagsterInvariantViolationError(
"Could not find schedule named {schedule_name}"
.format(schedule_name=schedule_name), )
external_schedule = external_repo.get_external_schedule(
schedule_name)
# Validate that either the schedule has no timezone or it matches
# the system timezone
schedule_timezone = external_schedule.execution_timezone
if schedule_timezone:
system_timezone = pendulum.now().timezone.name
if system_timezone != external_schedule.execution_timezone:
raise DagsterInvariantViolationError(
"Schedule {schedule_name} is set to execute in {schedule_timezone}, "
"but this scheduler can only run in the system timezone, "
"{system_timezone}. Use DagsterDaemonScheduler if you want to be able "
"to execute schedules in arbitrary timezones."
.format(
schedule_name=external_schedule.name,
schedule_timezone=schedule_timezone,
system_timezone=system_timezone,
), )
_launch_scheduled_executions(instance, repo_location,
external_repo,
external_schedule,
tick_context)
def grpc_command(
port=None,
socket=None,
host=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
ipc_output_file=None,
fixed_server_id=None,
override_system_timezone=None,
log_level="INFO",
use_python_environment_entry_point=False,
container_context=None,
**kwargs,
):
if seven.IS_WINDOWS and port is None:
raise click.UsageError(
"You must pass a valid --port/-p on Windows: --socket/-s not supported."
)
if not (port or socket and not (port and socket)):
raise click.UsageError(
"You must pass one and only one of --port/-p or --socket/-s.")
configure_loggers(log_level=coerce_valid_log_level(log_level))
logger = logging.getLogger("dagster.code_server")
loadable_target_origin = None
if any(kwargs[key] for key in [
"attribute",
"working_directory",
"module_name",
"package_name",
"python_file",
"empty_working_directory",
]):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
attribute=kwargs["attribute"],
working_directory=(None if kwargs.get("empty_working_directory")
else get_working_directory_from_kwargs(kwargs)),
module_name=kwargs["module_name"],
python_file=kwargs["python_file"],
package_name=kwargs["package_name"],
)
with (mock_system_timezone(override_system_timezone)
if override_system_timezone else nullcontext()):
server = DagsterGrpcServer(
port=port,
socket=socket,
host=host,
loadable_target_origin=loadable_target_origin,
max_workers=max_workers,
heartbeat=heartbeat,
heartbeat_timeout=heartbeat_timeout,
lazy_load_user_code=lazy_load_user_code,
ipc_output_file=ipc_output_file,
fixed_server_id=fixed_server_id,
entry_point=(get_python_environment_entry_point(sys.executable)
if use_python_environment_entry_point else
DEFAULT_DAGSTER_ENTRY_POINT),
container_context=json.loads(container_context)
if container_context != None else None,
)
code_desc = " "
if loadable_target_origin:
if loadable_target_origin.python_file:
code_desc = f" for file {loadable_target_origin.python_file} "
elif loadable_target_origin.package_name:
code_desc = f" for package {loadable_target_origin.package_name} "
elif loadable_target_origin.module_name:
code_desc = f" for module {loadable_target_origin.module_name} "
server_desc = (
f"Dagster code server{code_desc}on port {port} in process {os.getpid()}"
if port else
f"Dagster code server{code_desc}in process {os.getpid()}")
logger.info("Started {server_desc}".format(server_desc=server_desc))
try:
server.serve()
finally:
logger.info(
"Shutting down {server_desc}".format(server_desc=server_desc))
