def test_perform_prediction_step_put_task(scheduler: Scheduler): """Validates task serialisation and using UTC time.""" with patch.object(scheduler.task_queue, "send_event") as send_event_mock: scheduler.perform_prediction_step() from_datetime = DATETIME_UTC_NOW + timedelta(days=-SCHEDULER_PREDICTION_DELAY) from_time = from_datetime.timestamp() to_time = (from_datetime + timedelta(minutes=+SCHEDULER_PREDICTION_INTERVAL)).timestamp() dumped_task = '{{"project_name": "project", "model_name": "model", ' '"from_time": {}, "to_time": {}}}'.format( from_time, to_time ) send_event_mock.assert_called_once_with(dumped_task)
def test_run(scheduler: Scheduler): with patch.object(scheduler, "task_queue") as task_queue, patch.object( scheduler, "models_metadata_info_provider") as md_client: md_client.get_projects.return_value = ["project1"] scheduler.run() task_queue.assert_has_calls([ call.put_task( Task(project_name="project", model_name=ANY, from_time=ANY, to_time=ANY)) ])
def test_run_intervals_sleep(clock_sleep_mock, scheduler: Scheduler): """Test scheduling behavior of the Clock. If interval is 5 minutes between scheduling predictions, logic is: 10:01:00 - scheduler was run -> "sleep" for 4 mins; 10:05:00 - first prediction scheduling made; 10:10:00 - each next scheduling should be made in 5 mins; ... """ times_now = ["10:01:00", "10:05:20", "10:10:25", "10:16:13"] expected_sleeps = [240.0, 280.0, 275.0, 227.0] datetime_now = [ datetime.fromisoformat(f"2020-04-10T{t}.000000+00:00") for t in times_now ] with patch("latigo.clock.datetime") as mock_clock_dt, patch.object( scheduler, "_run", new=Mock()): mock_clock_dt.datetime.now.side_effect = datetime_now mock_clock_dt.date.side_effect = datetime_date mock_clock_dt.datetime.combine.side_effect = datetime.combine scheduler.run() clock_sleep_mock.assert_has_calls([call(secs) for secs in expected_sleeps])
def scheduler(schedule_config, monkeypatch) -> Scheduler: monkeypatch.setattr("latigo.scheduler.get_datetime_now_in_utc", MagicMock(return_value=DATETIME_UTC_NOW)) scheduler = Scheduler(schedule_config) scheduler.models = MODELS return scheduler
"auth": { "resource": environ.get("LATIGO_GORDO_RESOURCE", not_found), "tenant": environ.get("LATIGO_GORDO_TENANT", not_found), "authority_host_url": environ.get("LATIGO_GORDO_AUTH_HOST_URL", not_found), "client_id": environ.get("LATIGO_GORDO_CLIENT_ID", not_found), "client_secret": environ.get("LATIGO_GORDO_CLIENT_SECRET", not_found), }, }, } # fmt: on config_filename = environ.get("LATIGO_SCHEDULER_CONFIG_FILE", "scheduler_config.yaml") config = load_config(config_filename, config_overlay) if not config: logger.error( f"Could not load configuration for scheduler from {config_filename}") sys.exit(1) logger.info("Preparing Latigo Scheduler") scheduler = Scheduler(config) logger.info("Running Latigo Scheduler") scheduler.run() logger.info("Stopping Latigo Scheduler")
import distutils.util import os import socket import sys import threading from latigo.log import setup_logging from latigo import __version__ as latigo_version logger = setup_logging("latigo.app.scheduler") from latigo.utils import load_configs, sleep from latigo.scheduler import Scheduler config, err = load_configs( "../deploy/scheduler_config.yaml", os.environ["LATIGO_SCHEDULER_CONFIG_FILE"] or None, ) if not config: logger.error(f"Could not load configuration for scheduler: {err}") sleep(60 * 5) sys.exit(1) threading.current_thread().name = config.get("scheduler", {}).get( "instance_name", f"latigo-scheduler-{latigo_version}-{socket.getfqdn()}") logger.info("Configuring Latigo Scheduler") scheduler = Scheduler(config) scheduler.print_summary() logger.info("Running Latigo Scheduler") scheduler.run() logger.info("Stopping Latigo Scheduler")