async def test_startup_schedule(self):
"""Test startup of _scheduler
:assert:
the number of running tasks
"""
await self.populate_test_data(
) # Populate data in foglamp.scheduled_processes
scheduler = Scheduler(_address, _m_port)
await scheduler.start()
# Declare schedule startup, and execute
startup_schedule = StartUpSchedule(
) # A scheduled process of the _scheduler
startup_schedule.name = 'startup schedule'
startup_schedule.process_name = 'sleep30'
startup_schedule.repeat = datetime.timedelta(
seconds=0) # set no repeat to startup
await scheduler.save_schedule(startup_schedule)
await asyncio.sleep(1)
# Assert no tasks ar running
tasks = await scheduler.get_running_tasks()
assert len(tasks) == 0
await scheduler.get_schedule(startup_schedule.schedule_id
) # ID of the schedule startup
await self.stop_scheduler(scheduler)
scheduler = Scheduler()
await scheduler.start()
await asyncio.sleep(2)
# Assert only 1 task is running
tasks = await scheduler.get_running_tasks()
assert len(tasks) == 1
scheduler.max_running_tasks = 0 # set that no tasks would run
await scheduler.cancel_task(tasks[0].task_id)
await asyncio.sleep(2)
# Assert no tasks are running
tasks = await scheduler.get_running_tasks()
assert len(tasks) == 0
scheduler.max_running_tasks = 1
await asyncio.sleep(2)
# Assert a single task is running
tasks = await scheduler.get_running_tasks()
assert len(tasks) == 1
await self.stop_scheduler(scheduler)
async def mock_coro():
schedule = StartUpSchedule()
schedule.schedule_id = self._random_uuid
schedule.exclusive = True
schedule.enabled = True
schedule.name = "foo"
schedule.process_name = "bar"
schedule.repeat = timedelta(seconds=30)
schedule.time = None
schedule.day = None
return schedule
async def mock_coro():
schedules = []
schedule = StartUpSchedule()
schedule.schedule_id = "1"
schedule.exclusive = True
schedule.enabled = True
schedule.name = "foo"
schedule.process_name = "bar"
schedule.repeat = timedelta(seconds=30)
schedule.time = None
schedule.day = None
schedules.append(schedule)
return schedules
async def _execute_add_update_schedule(data, curr_value=None):
"""
Private method common to create a new schedule and update an existing schedule
Args:
data:
Returns:
schedule_id (new for created, existing for updated)
"""
_schedule = _extract_args(data, curr_value)
# Create schedule object as Scheduler.save_schedule requires an object
if _schedule.get('schedule_type') == Schedule.Type.STARTUP:
schedule = StartUpSchedule()
elif _schedule.get('schedule_type') == Schedule.Type.TIMED:
schedule = TimedSchedule()
schedule.day = _schedule.get('schedule_day')
m, s = divmod(_schedule.get('schedule_time'), 60)
h, m = divmod(m, 60)
schedule.time = datetime.time().replace(hour=h, minute=m, second=s)
elif _schedule.get('schedule_type') == Schedule.Type.INTERVAL:
schedule = IntervalSchedule()
elif _schedule.get('schedule_type') == Schedule.Type.MANUAL:
schedule = ManualSchedule()
# Populate scheduler object
schedule.schedule_id = _schedule.get('schedule_id')
schedule.name = _schedule.get('schedule_name')
schedule.process_name = _schedule.get('schedule_process_name')
schedule.repeat = datetime.timedelta(seconds=_schedule['schedule_repeat'])
schedule.exclusive = True if _schedule.get(
'schedule_exclusive') == 'True' else False
schedule.enabled = True if _schedule.get(
'schedule_enabled') == 'True' else False
# Save schedule
await server.Server.scheduler.save_schedule(
schedule, _schedule['is_enabled_modified'])
updated_schedule_id = schedule.schedule_id
return updated_schedule_id
async def async_mock_get_schedule():
schedule = StartUpSchedule()
schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b'
return schedule
async def async_mock_get_schedule():
schedule = StartUpSchedule()
schedule.schedule_id = sch_id
return schedule
def _schedule_row_to_schedule(cls, schedule_id: uuid.UUID,
schedule_row: _ScheduleRow) -> Schedule:
schedule_type = schedule_row.type
if schedule_type == Schedule.Type.STARTUP:
schedule = StartUpSchedule()
elif schedule_type == Schedule.Type.TIMED:
schedule = TimedSchedule()
elif schedule_type == Schedule.Type.INTERVAL:
schedule = IntervalSchedule()
elif schedule_type == Schedule.Type.MANUAL:
schedule = ManualSchedule()
else:
raise ValueError("Unknown schedule type {}", schedule_type)
schedule.schedule_id = schedule_id
schedule.exclusive = schedule_row.exclusive
schedule.name = schedule_row.name
schedule.process_name = schedule_row.process_name
schedule.repeat = schedule_row.repeat
if schedule_type == Schedule.Type.TIMED:
schedule.day = schedule_row.day
schedule.time = schedule_row.time
else:
schedule.day = None
schedule.time = None
return schedule
async def mock_schedule(*args):
schedule = StartUpSchedule()
schedule.schedule_id = self._random_uuid
schedule.exclusive = True
schedule.enabled = True
schedule.process_name = "bar"
schedule.repeat = timedelta(seconds=30)
schedule.time = None
schedule.day = None
if args[0] == 1:
schedule.name = "foo"
else:
schedule.name = "new"
return schedule
async def mock_schedule(_type):
if _type == 1:
schedule = StartUpSchedule()
schedule.repeat = None
schedule.time = None
schedule.day = None
elif _type == 2:
schedule = TimedSchedule()
schedule.repeat = None
schedule.time = datetime(1, 1, 1, 0, 0, 10)
schedule.day = 1
elif _type == 3:
schedule = IntervalSchedule()
schedule.repeat = timedelta(seconds=15)
schedule.time = None
schedule.day = None
else:
schedule = ManualSchedule()
schedule.repeat = None
schedule.time = None
schedule.day = None
schedule.schedule_id = self._random_uuid
schedule.exclusive = True
schedule.enabled = True
schedule.name = "foo"
schedule.process_name = "bar"
return schedule
async def add_service(request):
"""
Create a new service to run a specific plugin
:Example:
curl -X POST http://localhost:8081/foglamp/service -d '{"name": "DHT 11", "plugin": "dht11", "type": "south", "enabled": true}'
curl -sX POST http://localhost:8081/foglamp/service -d '{"name": "Sine", "plugin": "sinusoid", "type": "south", "enabled": true, "config": {"dataPointsPerSec": {"value": "10"}}}' | jq
curl -X POST http://localhost:8081/foglamp/service -d '{"name": "NotificationServer", "type": "notification", "enabled": true}' | jq
"""
try:
data = await request.json()
if not isinstance(data, dict):
raise ValueError('Data payload must be a valid JSON')
name = data.get('name', None)
plugin = data.get('plugin', None)
service_type = data.get('type', None)
enabled = data.get('enabled', None)
config = data.get('config', None)
if name is None:
raise web.HTTPBadRequest(
reason='Missing name property in payload.')
if utils.check_reserved(name) is False:
raise web.HTTPBadRequest(
reason='Invalid name property in payload.')
if utils.check_foglamp_reserved(name) is False:
raise web.HTTPBadRequest(
reason=
"'{}' is reserved for FogLAMP and can not be used as service name!"
.format(name))
if service_type is None:
raise web.HTTPBadRequest(
reason='Missing type property in payload.')
service_type = str(service_type).lower()
if service_type == 'north':
raise web.HTTPNotAcceptable(
reason='north type is not supported for the time being.')
if service_type not in ['south', 'notification']:
raise web.HTTPBadRequest(
reason='Only south and notification type are supported.')
if plugin is None and service_type == 'south':
raise web.HTTPBadRequest(
reason='Missing plugin property for type south in payload.')
if plugin and utils.check_reserved(plugin) is False:
raise web.HTTPBadRequest(
reason='Invalid plugin property in payload.')
if enabled is not None:
if enabled not in ['true', 'false', True, False]:
raise web.HTTPBadRequest(
reason='Only "true", "false", true, false'
' are allowed for value of enabled.')
is_enabled = True if (
(type(enabled) is str and enabled.lower() in ['true']) or
((type(enabled) is bool and enabled is True))) else False
# Check if a valid plugin has been provided
plugin_module_path, plugin_config, process_name, script = "", {}, "", ""
if service_type == 'south':
# "plugin_module_path" is fixed by design. It is MANDATORY to keep the plugin in the exactly similar named
# folder, within the plugin_module_path.
# if multiple plugin with same name are found, then python plugin import will be tried first
plugin_module_path = "foglamp.plugins.south"
try:
plugin_info = load_python_plugin(plugin_module_path, plugin,
service_type)
plugin_config = plugin_info['config']
if not plugin_config:
_logger.exception("Plugin %s import problem from path %s",
plugin, plugin_module_path)
raise web.HTTPNotFound(
reason='Plugin "{}" import problem from path "{}".'.
format(plugin, plugin_module_path))
process_name = 'south_c'
script = '["services/south_c"]'
except ImportError as ex:
# Checking for C-type plugins
plugin_config = load_c_plugin(plugin, service_type)
if not plugin_config:
_logger.exception(
"Plugin %s import problem from path %s. %s", plugin,
plugin_module_path, str(ex))
raise web.HTTPNotFound(
reason='Plugin "{}" import problem from path "{}".'.
format(plugin, plugin_module_path))
process_name = 'south_c'
script = '["services/south_c"]'
except TypeError as ex:
_logger.exception(str(ex))
raise web.HTTPBadRequest(reason=str(ex))
except Exception as ex:
_logger.exception("Failed to fetch plugin configuration. %s",
str(ex))
raise web.HTTPInternalServerError(
reason='Failed to fetch plugin configuration')
elif service_type == 'notification':
process_name = 'notification_c'
script = '["services/notification_c"]'
storage = connect.get_storage_async()
config_mgr = ConfigurationManager(storage)
# Check whether category name already exists
category_info = await config_mgr.get_category_all_items(
category_name=name)
if category_info is not None:
raise web.HTTPBadRequest(
reason="The '{}' category already exists".format(name))
# Check that the schedule name is not already registered
count = await check_schedules(storage, name)
if count != 0:
raise web.HTTPBadRequest(
reason='A service with this name already exists.')
# Check that the process name is not already registered
count = await check_scheduled_processes(storage, process_name)
if count == 0:
# Now first create the scheduled process entry for the new service
payload = PayloadBuilder().INSERT(name=process_name,
script=script).payload()
try:
res = await storage.insert_into_tbl("scheduled_processes",
payload)
except StorageServerError as ex:
_logger.exception("Failed to create scheduled process. %s",
ex.error)
raise web.HTTPInternalServerError(
reason='Failed to create service.')
except Exception as ex:
_logger.exception("Failed to create scheduled process. %s",
str(ex))
raise web.HTTPInternalServerError(
reason='Failed to create service.')
# check that notification service is not already registered, right now notification service LIMIT to 1
if service_type == 'notification':
res = await check_notification_schedule(storage)
for ps in res['rows']:
if 'notification_c' in ps['process_name']:
raise web.HTTPBadRequest(
reason='A Notification service schedule already exists.'
)
elif service_type == 'south':
try:
# Create a configuration category from the configuration defined in the plugin
category_desc = plugin_config['plugin']['description']
await config_mgr.create_category(
category_name=name,
category_description=category_desc,
category_value=plugin_config,
keep_original_items=True)
# Create the parent category for all South services
await config_mgr.create_category("South", {},
"South microservices", True)
await config_mgr.create_child_category("South", [name])
# If config is in POST data, then update the value for each config item
if config is not None:
if not isinstance(config, dict):
raise ValueError('Config must be a JSON object')
for k, v in config.items():
await config_mgr.set_category_item_value_entry(
name, k, v['value'])
except Exception as ex:
await config_mgr.delete_category_and_children_recursively(name)
_logger.exception("Failed to create plugin configuration. %s",
str(ex))
raise web.HTTPInternalServerError(
reason='Failed to create plugin configuration.')
# If all successful then lastly add a schedule to run the new service at startup
try:
schedule = StartUpSchedule()
schedule.name = name
schedule.process_name = process_name
schedule.repeat = datetime.timedelta(0)
schedule.exclusive = True
# if "enabled" is supplied, it gets activated in save_schedule() via is_enabled flag
schedule.enabled = False
# Save schedule
await server.Server.scheduler.save_schedule(schedule, is_enabled)
schedule = await server.Server.scheduler.get_schedule_by_name(name)
except StorageServerError as ex:
await config_mgr.delete_category_and_children_recursively(name)
_logger.exception("Failed to create schedule. %s", ex.error)
raise web.HTTPInternalServerError(
reason='Failed to create service.')
except Exception as ex:
await config_mgr.delete_category_and_children_recursively(name)
_logger.exception("Failed to create service. %s", str(ex))
raise web.HTTPInternalServerError(
reason='Failed to create service.')
except ValueError as e:
raise web.HTTPBadRequest(reason=str(e))
else:
return web.json_response({
'name': name,
'id': str(schedule.schedule_id)
})
async def add_service(request):
"""
Create a new service to run a specific plugin
:Example:
curl -X POST /foglamp/service -d '{"name": "furnace4", "type": "south", "plugin": "dht11"}'
"""
try:
data = await request.json()
if not isinstance(data, dict):
raise ValueError('Data payload must be a dictionary')
name = data.get('name', None)
plugin = data.get('plugin', None)
service_type = data.get('type', None)
if name is None:
raise web.HTTPBadRequest(
reason='Missing name property in payload.')
if plugin is None:
raise web.HTTPBadRequest(
reason='Missing plugin property in payload.')
if service_type is None:
raise web.HTTPBadRequest(
reason='Missing type property in payload.')
if not service_type in ['south', 'north']:
raise web.HTTPBadRequest(
reason='Only north and south types are supported.')
storage = connect.get_storage()
# Check that the process is not already registered
payload = PayloadBuilder().SELECT("name").WHERE(['name', '=',
name]).payload()
result = storage.query_tbl_with_payload('scheduled_processes', payload)
count = result['count']
if count != 0:
raise web.HTTPBadRequest(
reason='A service with that name already exists')
# First create the scheduled process entry for our new service
if service_type == 'south':
script = '["services/south"]'
if service_type == 'north':
script = '["services/north"]'
payload = PayloadBuilder().INSERT(name=name, script=script).payload()
try:
res = storage.insert_into_tbl("scheduled_processes", payload)
except Exception as ins_ex:
raise web.HTTPInternalServerError(
reason='Failed to created scheduled process. {}'.format(
str(ins_ex)))
# Now create a configuration category with the minimum to load the plugin
# TODO It would be better to load the default configuration from the plugin
# and use this, however we should extract the plugin loading code so that is shared
new_category = {
"plugin": {
"type": "string",
"default": plugin,
"description": "Python module name of the plugin to load"
}
}
category_desc = '{} service configuration'.format(name)
config_mgr = ConfigurationManager(storage)
await config_mgr.create_category(category_name=name,
category_description=category_desc,
category_value=new_category,
keep_original_items=False)
# Check that the process is not already registered
payload = PayloadBuilder().SELECT("schedule_name").WHERE(
['schedule_name', '=', name]).payload()
result = storage.query_tbl_with_payload('schedules', payload)
count = result['count']
if count != 0:
raise web.HTTPBadRequest(
reason='A schedule with that name already exists')
# Finally add a schedule to run the new service at startup
schedule = StartUpSchedule()
schedule.name = name
schedule.process_name = name
schedule.repeat = datetime.timedelta(0)
schedule.exclusive = True
schedule.enabled = False
# Save schedule
await server.Server.scheduler.save_schedule(schedule)
schedule = await server.Server.scheduler.get_schedule_by_name(name)
return web.json_response({
'name': name,
'id': str(schedule.schedule_id)
})
except ValueError as ex:
raise web.HTTPNotFound(reason=str(ex))
async def add_service(request):
"""
Create a new service to run a specific plugin
:Example:
curl -X POST /foglamp/service -d '{"name": "furnace4", "type": "south", "plugin": "dht11"}'
"""
try:
data = await request.json()
if not isinstance(data, dict):
raise ValueError('Data payload must be a dictionary')
name = data.get('name', None)
plugin = data.get('plugin', None)
service_type = data.get('type', None)
enabled = data.get('enabled', None)
if name is None:
raise web.HTTPBadRequest(
reason='Missing name property in payload.')
if plugin is None:
raise web.HTTPBadRequest(
reason='Missing plugin property in payload.')
if service_type is None:
raise web.HTTPBadRequest(
reason='Missing type property in payload.')
if not service_type in ['south', 'north']:
raise web.HTTPBadRequest(
reason='Only north and south types are supported.')
if enabled is not None:
if enabled not in ['t', 'f', 'true', 'false', 0, 1]:
raise web.HTTPBadRequest(
reason=
'Only "t", "f", "true", "false" are allowed for value of enabled.'
)
is_enabled = True if (
(type(enabled) is str and enabled.lower() in ['t', 'true']) or
((type(enabled) is bool and enabled is True))) else False
storage = connect.get_storage_async()
# Check that the process name is not already registered
payload = PayloadBuilder().SELECT("name").WHERE(['name', '=',
name]).payload()
result = await storage.query_tbl_with_payload('scheduled_processes',
payload)
count = result['count']
if count != 0:
raise web.HTTPBadRequest(
reason='A service with that name already exists')
# Check that the schedule name is not already registered
payload = PayloadBuilder().SELECT("schedule_name").WHERE(
['schedule_name', '=', name]).payload()
result = await storage.query_tbl_with_payload('schedules', payload)
count = result['count']
if count != 0:
raise web.HTTPBadRequest(
reason='A schedule with that name already exists')
# First create the scheduled process entry for our new service
if service_type == 'south':
script = '["services/south"]'
plugin_module_path = "foglamp.plugins.south"
if service_type == 'north':
script = '["services/north"]'
plugin_module_path = "foglamp.plugins.north"
payload = PayloadBuilder().INSERT(name=name, script=script).payload()
try:
res = await storage.insert_into_tbl("scheduled_processes", payload)
except Exception as ins_ex:
raise web.HTTPInternalServerError(
reason='Failed to created scheduled process. {}'.format(
str(ins_ex)))
# Now load the plugin to fetch its configuration
try:
# "plugin_module_path" is fixed by design. It is MANDATORY to keep the plugin in the exactly similar named
# folder, within the plugin_module_path.
import_file_name = "{path}.{dir}.{file}".format(
path=plugin_module_path, dir=plugin, file=plugin)
_plugin = __import__(import_file_name, fromlist=[''])
# Fetch configuration from the configuration defined in the plugin
plugin_info = _plugin.plugin_info()
plugin_config = plugin_info['config']
# Create a configuration category from the configuration defined in the plugin
category_desc = plugin_config['plugin']['description']
config_mgr = ConfigurationManager(storage)
await config_mgr.create_category(
category_name=name,
category_description=category_desc,
category_value=plugin_config,
keep_original_items=True)
except ImportError as ex:
raise web.HTTPInternalServerError(
reason='Plugin "{}" import problem from path "{}". {}'.format(
plugin, plugin_module_path, str(ex)))
except Exception as ex:
raise web.HTTPInternalServerError(
reason='Failed to create plugin configuration. {}'.format(
str(ex)))
# Next add a schedule to run the new service at startup
schedule = StartUpSchedule() # TODO: For North plugin also?
schedule.name = name
schedule.process_name = name
schedule.repeat = datetime.timedelta(0)
schedule.exclusive = True
schedule.enabled = False # if "enabled" is supplied, it gets activated in save_schedule() via is_enabled flag
# Save schedule
await server.Server.scheduler.save_schedule(schedule, is_enabled)
schedule = await server.Server.scheduler.get_schedule_by_name(name)
return web.json_response({
'name': name,
'id': str(schedule.schedule_id)
})
except ValueError as ex:
raise web.HTTPNotFound(reason=str(ex))