def test_create_normal_scheduling_no_estimate_provided(self):
input_provider = Mock()
input_provider.sched_params = SchedulerParameters()
with patch('adaptive_scheduler.scheduler_input.SchedulingInputFactory._create_scheduling_input',
self.create_input_mock, create=True):
factory = SchedulingInputFactory(input_provider)
factory.create_normal_scheduling_input()
assert_equal(1, input_provider.set_normal_mode.call_count)
assert_equal(0, input_provider.set_normal_run_time.call_count)
def _schedule_requests(self,
rr_rg_list,
normal_rg_list,
scheduler_time,
rr_loop=False,
block_schedule_by_resource=None,
running_request_groups=None,
rapid_response_ids=None,
semester_details=None):
if block_schedule_by_resource is None:
block_schedule_by_resource = {}
if running_request_groups is None:
running_request_groups = []
if rapid_response_ids is None:
rapid_response_ids = []
if semester_details is None:
semester_details = {}
sched_params = SchedulerParameters(run_once=True,
dry_run=True,
timelimit_seconds=30)
event_bus_mock = Mock()
scheduler = LCOGTNetworkScheduler(FullScheduler_ortoolkit,
sched_params, event_bus_mock,
self.telescopes)
network_interface_mock = Mock()
network_interface_mock.cancel = Mock(return_value=0)
network_interface_mock.save = Mock(return_value=0)
network_interface_mock.abort = Mock(return_value=0)
network_interface_mock.get_current_events = Mock(return_value={})
mock_input_factory = create_scheduler_input_factory(
rr_rg_list, normal_rg_list, block_schedule_by_resource,
running_request_groups, rapid_response_ids)
if rr_loop:
scheduler_input = mock_input_factory.create_rr_scheduling_input()
else:
scheduler_input = mock_input_factory.create_normal_scheduling_input(
)
scheduler_input.scheduler_now = scheduler_time
scheduler_input.estimated_scheduler_end = scheduler_time + timedelta(
minutes=15)
if not semester_details:
semester_details = {
'id': '2015A',
'start': scheduler_time - timedelta(days=150),
'end': scheduler_time + timedelta(days=150)
}
result = scheduler.run_scheduler(scheduler_input,
scheduler_time +
timedelta(minutes=15),
semester_details,
preemption_enabled=rr_loop)
return result
def test_multiple_rr_has_correct_cancel_date_list(self):
''' Schedules three nearly back to back RRs. Checks that each of their scheduled time appears in the date list
for the resource they are scheduled in when getting dates to cancel.
'''
scheduler_start = self.base_time - timedelta(hours=10)
result = self._schedule_requests(
[self.rr_request_group_2, self.rr_request_group_1], [
self.many_request_group_2,
],
scheduler_start,
rr_loop=True,
block_schedule_by_resource={})
scheduled_rgs = result.get_scheduled_requests_by_request_group_id()
assert_true(5 in scheduled_rgs)
assert_true(5 in scheduled_rgs[5])
assert_true(6 in scheduled_rgs)
assert_true(3 in scheduled_rgs[6])
assert_true(1 in scheduled_rgs[6])
semester_start = scheduler_start - timedelta(days=150)
scheduler_runner = SchedulerRunner(SchedulerParameters(dry_run=True),
Mock(), Mock(), Mock(), Mock())
scheduler_runner.semester_details = {
'id': '2015A',
'start': semester_start,
'end': scheduler_start + timedelta(days=150)
}
cancel_date_list_by_resource = scheduler_runner._determine_schedule_cancelation_list_from_new_schedule(
result.schedule)
assert_true('1m0a.doma.ogg' in cancel_date_list_by_resource)
assert_equal(len(cancel_date_list_by_resource['1m0a.doma.ogg']), 3)
assert_equal(len(cancel_date_list_by_resource['1m0a.doma.ogg']), 3)
dt_start, dt_end = get_reservation_datetimes(scheduled_rgs[6][1],
semester_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][0][0],
dt_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][0][1],
dt_end)
dt_start, dt_end = get_reservation_datetimes(scheduled_rgs[6][3],
semester_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][1][0],
dt_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][1][1],
dt_end)
dt_start, dt_end = get_reservation_datetimes(scheduled_rgs[5][5],
semester_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][2][0],
dt_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][2][1],
dt_end)
def test_create_rr_scheduling(self):
input_provider = Mock()
input_provider.sched_params = SchedulerParameters()
input_provider.json_request_group_list = []
with patch('adaptive_scheduler.scheduler_input.SchedulingInputFactory._create_scheduling_input',
self.create_input_mock, create=True):
factory = SchedulingInputFactory(input_provider)
factory.create_rr_scheduling_input(100)
assert_equal(1, input_provider.set_rr_mode.call_count)
assert_equal(1, input_provider.set_rr_run_time.call_count)
assert_equal(100, input_provider.set_rr_run_time.call_args[0][0])
def setUp(self):
self.sched_params = SchedulerParameters()
self.network_interface = Mock()
self.network_interface.get_all_request_groups = Mock(return_value=[])
self.observation_portal_interface = Mock()
self.observation_portal_interface.get_semester_details = Mock(
return_value={
'id': '2015A',
'start': datetime.utcnow() - timedelta(days=30),
'end': datetime.utcnow() + timedelta(days=30)
})
self.network_interface.observation_portal_interface = self.observation_portal_interface
self.network_model = {}
def test_changing_semester_details_clears_visibility_cache(self):
scheduler_time = self.base_time - timedelta(hours=10)
sched_params = SchedulerParameters(run_once=True,
dry_run=True,
timelimit_seconds=30)
event_bus_mock = Mock()
scheduler = LCOGTNetworkScheduler(FullScheduler_ortoolkit,
sched_params, event_bus_mock,
self.telescopes)
network_interface_mock = Mock()
network_interface_mock.cancel = Mock(return_value=0)
network_interface_mock.save = Mock(return_value=0)
network_interface_mock.abort = Mock(return_value=0)
network_interface_mock.get_current_events = Mock(return_value={})
normal_ur_list = [self.and_request_group_1, self.and_request_group_2]
mock_input_factory = create_scheduler_input_factory([], normal_ur_list,
{}, [], [])
scheduler_input = mock_input_factory.create_normal_scheduling_input()
scheduler_input.scheduler_now = scheduler_time
scheduler_input.estimated_scheduler_end = scheduler_time + timedelta(
minutes=15)
semester_details = {
'id': '2015A',
'start': scheduler_time - timedelta(days=150),
'end': scheduler_time + timedelta(days=150)
}
scheduler.run_scheduler(scheduler_input,
scheduler_time + timedelta(minutes=15),
semester_details,
preemption_enabled=False)
assert scheduler.visibility_cache != {}
saved_visibility_cache = scheduler.visibility_cache
# Now run again with a different semester to clear visibility cache
semester_details['start'] = scheduler_time - timedelta(days=149)
scheduler.run_scheduler(scheduler_input,
scheduler_time + timedelta(minutes=15),
semester_details,
preemption_enabled=False)
assert scheduler.visibility_cache != {}
assert scheduler.visibility_cache != saved_visibility_cache
def test_one_rr_has_correct_cancel_date_list(self):
''' Schedules a single RR and verifies it's time appears in the cancellation date list on the resource
'''
scheduler_start = self.base_time - timedelta(hours=10)
result = self._schedule_requests([
self.rr_request_group_1,
], [
self.many_request_group_2,
],
scheduler_start,
rr_loop=True,
block_schedule_by_resource={})
scheduled_rgs = result.get_scheduled_requests_by_request_group_id()
assert_true(5 in scheduled_rgs)
assert_true(5 in scheduled_rgs[5])
semester_start = scheduler_start - timedelta(days=150)
dt_start, dt_end = get_reservation_datetimes(scheduled_rgs[5][5],
semester_start)
scheduler_runner = SchedulerRunner(SchedulerParameters(dry_run=True),
Mock(), Mock(), Mock(), Mock())
scheduler_runner.semester_details = {
'id': '2015A',
'start': semester_start,
'end': scheduler_start + timedelta(days=150)
}
cancel_date_list_by_resource = scheduler_runner._determine_schedule_cancelation_list_from_new_schedule(
result.schedule)
assert_true('1m0a.doma.ogg' in cancel_date_list_by_resource)
assert_equal(len(cancel_date_list_by_resource['1m0a.doma.ogg']), 1)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][0][0],
dt_start)
assert_equal(cancel_date_list_by_resource['1m0a.doma.ogg'][0][1],
dt_end)
def parse_args(argv):
defaults = SchedulerParameters()
arg_parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__)
arg_parser.add_argument("-l", "--timelimit", type=float, default=defaults.timelimit_seconds,
dest='timelimit_seconds',
help="The time limit of the scheduler kernel, in seconds; negative implies no limit")
arg_parser.add_argument("-i", "--horizon", type=float, default=defaults.horizon_days, dest='horizon_days',
help="The scheduler's horizon, in days")
arg_parser.add_argument("-z", "--slicesize", type=int, default=defaults.slicesize_seconds, dest='slicesize_seconds',
help="The discretization size of the scheduler, in seconds")
arg_parser.add_argument("-s", "--sleep", type=int, default=defaults.sleep_seconds, dest='sleep_seconds',
help="Sleep period between scheduling runs, in seconds")
arg_parser.add_argument("-p", "--observation_portal_url", type=str, dest='observation_portal_url',
help="Observation Portal base URL", default=defaults.observation_portal_url)
arg_parser.add_argument("-c", "--configdb_url", type=str, dest='configdb_url', default=defaults.configdb_url,
help="ConfigDB endpoint URL")
arg_parser.add_argument("-d", "--dry-run", type=bool, default=defaults.dry_run,
help="Perform a trial run with no changes made")
arg_parser.add_argument("-n", "--now", type=str, dest='simulate_now',
help="Alternative datetime to use as 'now', for running simulations (in isoformat: %%Y-%%m-%%dT%%H:%%M:%%SZ)")
arg_parser.add_argument("-w", "--noweather", type=bool, default=defaults.no_weather, dest='no_weather',
help="Disable weather checking")
arg_parser.add_argument("--nosingles", type=bool, default=defaults.no_singles, dest='no_singles',
help="Ignore the 'single' Request type")
arg_parser.add_argument("--nocompounds", type=bool, default=defaults.no_compounds, dest='no_compounds',
help="Ignore the 'and', 'oneof' and 'many' Request types")
arg_parser.add_argument("--no_rr", type=bool, default=defaults.no_rr, dest='no_rr',
help="Treat Rapid Response Requests like Normal Requests")
arg_parser.add_argument("--warm_starts", type=bool, default=defaults.warm_starts, dest='warm_starts',
help="Enable using warm start solutions in the scheduling kernel")
arg_parser.add_argument("-o", "--run-once", type=bool, default=defaults.run_once,
help="Only run the scheduling loop once, then exit")
arg_parser.add_argument("-k", "--kernel", type=str, default=defaults.kernel, choices=ALGORITHMS.keys(),
help="Options are GUROBI, CBC, GLPK, or SCIP. Default is SCIP")
arg_parser.add_argument("-f", "--fromfile", type=str, dest='input_file_name', default=defaults.input_file_name,
help="Filename for scheduler input. Example: -f scheduling_input_20180101.pickle")
arg_parser.add_argument("-g", "--mip_gap", type=float, default=defaults.mip_gap,
help="The acceptable MIP GAP threshold used in the solver. Defaults to 0.01 (1%). Recommended range 0.01-0.0001")
arg_parser.add_argument("--pickle", type=bool, default=defaults.pickle, dest='pickle',
help="Enable storing pickled files of scheduling run input")
arg_parser.add_argument("--save_output", type=bool, default=defaults.save_output, dest='save_output',
help="Enable storing scheduling run output in a json file")
arg_parser.add_argument("--request_logs", type=bool, default=defaults.request_logs, dest='request_logs',
help="Enable saving the per-request log files")
arg_parser.add_argument("--telescope_classes", type=str, default=','.join(defaults.telescope_classes),
help="Only schedule observations on the specified telescope_classes. Expects 3 character telescope classes comma delimited. If not specified, default is all classes.")
arg_parser.add_argument("--downtime_url", type=str, dest='downtime_url',
help="Downtime endpoint url", default=defaults.downtime_url)
arg_parser.add_argument("--elasticsearch_url", type=str, dest='elasticsearch_url',
help="Elasticsearch telemetry endpoint url", default=defaults.elasticsearch_url)
arg_parser.add_argument("--elasticsearch_index", type=str, dest='elasticsearch_index',
help="Elasticsearch telemetry index name", default=defaults.elasticsearch_index)
arg_parser.add_argument("--elasticsearch_excluded_observatories", type=str,
dest='elasticsearch_excluded_observatories',
help="Elasticsearch telemetry observatories to exclude (comma delimited)",
default=','.join(defaults.elasticsearch_excluded_observatories))
arg_parser.add_argument("--profiling_enabled", type=bool, dest='profiling_enabled',
help="Enable profiling output", default=defaults.profiling_enabled)
arg_parser.add_argument("--reservation_save_time_seconds", type=float, dest='avg_reservation_save_time_seconds',
help="Initial estimate for time needed to save a new scheduler reservation",
default=defaults.avg_reservation_save_time_seconds)
arg_parser.add_argument("--normal_runtime_seconds", type=float, dest='normal_runtime_seconds',
help="Initial estimate for the normal loop runtime",
default=defaults.normal_runtime_seconds)
arg_parser.add_argument("--rr_runtime_seconds", type=float, dest='rr_runtime_seconds',
help="Initial estimate for the Rapid Response loop runtime",
default=defaults.rr_runtime_seconds)
arg_parser.add_argument("--ignore_ipp", type=bool, dest='ignore_ipp',
help="Ignore intra-proposal priority when computing request priority",
default=defaults.ignore_ipp)
# Handle command line arguments
args, unknown = arg_parser.parse_known_args(argv)
if args.dry_run:
log.info("Running in simulation mode - no DB changes will be made")
log.info("Sleep period between scheduling runs set at %ds" % args.sleep_seconds)
rg_logger.disabled = not args.request_logs
sched_params = SchedulerParameters(**vars(args))
return sched_params
def parse_args(argv):
defaults = SchedulerParameters()
arg_parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__)
arg_parser.add_argument(
"-l",
"--timelimit",
type=float,
default=defaults.timelimit_seconds,
dest='timelimit_seconds',
help=
"The time limit of the scheduler kernel, in seconds; negative implies no limit"
)
arg_parser.add_argument("-i",
"--horizon",
type=float,
default=defaults.horizon_days,
dest='horizon_days',
help="The scheduler's horizon, in days")
arg_parser.add_argument(
"-z",
"--slicesize",
type=int,
default=defaults.slicesize_seconds,
dest='slicesize_seconds',
help="The discretization size of the scheduler, in seconds")
arg_parser.add_argument(
"-s",
"--sleep",
type=int,
default=defaults.sleep_seconds,
dest='sleep_seconds',
help="Sleep period between scheduling runs, in seconds")
arg_parser.add_argument("-p",
"--observation_portal_url",
type=str,
required=True,
dest='observation_portal_url',
help="Observation Portal base URL")
arg_parser.add_argument("-c",
"--configdb_url",
type=str,
dest='configdb_url',
default=defaults.configdb_url,
help="ConfigDB endpoint URL")
arg_parser.add_argument("-d",
"--dry-run",
action="store_true",
help="Perform a trial run with no changes made")
arg_parser.add_argument(
"-n",
"--now",
type=str,
dest='simulate_now',
help=
"Alternative datetime to use as 'now', for running simulations (in isoformat: %%Y-%%m-%%dT%%H:%%M:%%SZ)"
)
arg_parser.add_argument("-w",
"--noweather",
action="store_true",
dest='no_weather',
help="Disable weather checking")
arg_parser.add_argument("--nosingles",
action="store_true",
dest='no_singles',
help="Ignore the 'single' Request type")
arg_parser.add_argument(
"--nocompounds",
action="store_true",
dest='no_compounds',
help="Ignore the 'and', 'oneof' and 'many' Request types")
arg_parser.add_argument(
"--no_rr",
action="store_true",
dest='no_rr',
help="Treat Rapid Response Requests like Normal Requests")
arg_parser.add_argument(
"-o",
"--run-once",
action="store_true",
help="Only run the scheduling loop once, then exit")
arg_parser.add_argument(
"-k",
"--kernel",
type=str,
default=defaults.kernel,
help="Options are v5, v6, gurobi, mock. Default is gurobi")
arg_parser.add_argument(
"-f",
"--fromfile",
type=str,
dest='input_file_name',
default=defaults.input_file_name,
help=
"Filename for scheduler input. Example: -f scheduling_input_20180101.pickle"
)
arg_parser.add_argument(
"--pickle",
action="store_true",
dest='pickle',
help="Enable storing pickled files of scheduling run input")
arg_parser.add_argument(
"--save_output",
action="store_true",
dest='save_output',
help="Enable storing scheduling run output in a json file")
arg_parser.add_argument("--request_logs",
action="store_true",
dest='request_logs',
help="Enable saving the per-request log files")
arg_parser.add_argument("--downtime_url",
type=str,
dest='downtime_url',
help="Downtime endpoint url",
default=defaults.downtime_url)
arg_parser.add_argument("--elasticsearch_url",
type=str,
dest='elasticsearch_url',
help="Elasticsearch telemetry endpoint url",
default=defaults.elasticsearch_url)
arg_parser.add_argument("--elasticsearch_index",
type=str,
dest='elasticsearch_index',
help="Elasticsearch telemetry index name",
default=defaults.elasticsearch_index)
arg_parser.add_argument(
"--elasticsearch_excluded_observatories",
type=str,
dest='elasticsearch_excluded_observatories',
help=
"Elasticsearch telemetry observatories to exclude (comma delimited)",
default=defaults.elasticsearch_excluded_observatories)
arg_parser.add_argument("--profiling_enabled",
type=bool,
dest='profiling_enabled',
help="Enable profiling output",
default=defaults.profiling_enabled)
arg_parser.add_argument(
"--reservation_save_time_seconds",
type=float,
dest='avg_reservation_save_time_seconds',
help=
"Initial estimate for time needed to save a new scheduler reservation",
default=defaults.avg_reservation_save_time_seconds)
arg_parser.add_argument(
"--normal_runtime_seconds",
type=float,
dest='normal_runtime_seconds',
help="Initial estimate for the normal loop runtime",
default=defaults.normal_runtime_seconds)
arg_parser.add_argument(
"--rr_runtime_seconds",
type=float,
dest='rr_runtime_seconds',
help="Initial estimate for the Rapid Response loop runtime",
default=defaults.rr_runtime_seconds)
arg_parser.add_argument(
"--ignore_ipp",
action="store_true",
dest='ignore_ipp',
help="Ignore intra-proposal priority when computing request priority",
default=defaults.ignore_ipp)
arg_parser.add_argument(
"--debug",
action="store_true",
dest='debug',
help=
"Sets debug mode in the requestdb client calls to save error output to a file.",
default=defaults.debug)
# Handle command line arguments
args, unknown = arg_parser.parse_known_args(argv)
if args.dry_run:
log.info("Running in simulation mode - no DB changes will be made")
log.info("Sleep period between scheduling runs set at %ds" %
args.sleep_seconds)
rg_logger.disabled = not args.request_logs
sched_params = SchedulerParameters(**vars(args))
return sched_params
