async def test_purge_tasks(self): await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() interval_schedule = IntervalSchedule() interval_schedule.name = 'purge_task' interval_schedule.process_name = "sleep5" interval_schedule.repeat = datetime.timedelta(seconds=0) # interval_schedule.repeat = datetime.timedelta(seconds=30) await scheduler.save_schedule(interval_schedule) await asyncio.sleep(1) tasks = await scheduler.get_tasks(5) assert tasks scheduler.max_running_tasks = 0 await asyncio.sleep(7) scheduler.max_completed_task_age = datetime.timedelta(seconds=1) await scheduler.purge_tasks() tasks = await scheduler.get_tasks(5) assert not tasks await self.stop_scheduler(scheduler)
async def test_update(self): """Test update of a running task :assert: the number of tasks running information regarding the process running """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() interval_schedule = IntervalSchedule() interval_schedule.name = 'sleep10' interval_schedule.process_name = "sleep10" interval_schedule.repeat = datetime.timedelta(seconds=0) interval_schedule.enabled = True await scheduler.save_schedule(interval_schedule ) # Save update on _scheduler await asyncio.sleep(1) # Assert only 1 task is running tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 # Update 'updated' schedule interval interval_schedule.name = 'updated' interval_schedule.process_name = "sleep1" interval_schedule.repeat = datetime.timedelta( seconds=5) # Set time interval to 5 sec interval_schedule.enabled = True await scheduler.save_schedule(interval_schedule ) # Save update on _scheduler await asyncio.sleep(6) # Assert: only 1 task is running tasks = await scheduler.get_running_tasks( ) # list of current running tasks assert len(tasks) == 1 interval_schedule.exclusive = False await scheduler.save_schedule(interval_schedule) # Check able to get same schedule after restart # Check fields have been modified await self.stop_scheduler(scheduler) scheduler = Scheduler(_address, _m_port) await scheduler.start() schedule = await scheduler.get_schedule(interval_schedule.schedule_id) # Make sure that the values used by schedule are as expected assert schedule.process_name == 'sleep1' assert schedule.name == 'updated' assert schedule.repeat.seconds == 5 assert not schedule.exclusive await self.stop_scheduler(scheduler)
async def test_max_processes(self): """Test the maximum number of running processes :assert: the number of running processes """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # 2 maximum tasks # 1 runs at 1 second # 2 runs at 2 seconds # 3 runs at 11 seconds # 4 runs at 12 seconds # 5 runs at 21 seconds # 6 runs at 22 seconds # 7 runs at 31 seconds # 8 runs at 32 seconds # Total: 6 scheduler.max_running_tasks = 2 # set the maximum number of running tasks in parallel # Set interval schedule configuration interval_schedule = IntervalSchedule() interval_schedule.repeat = datetime.timedelta(seconds=1) interval_schedule.name = 'max active' interval_schedule.exclusive = False interval_schedule.process_name = 'sleep10' interval_schedule.enabled = True await scheduler.save_schedule(interval_schedule) await asyncio.sleep(30.3) scheduler.max_running_tasks = 0 # set the maximum number of running tasks in parallel tasks = await scheduler.get_tasks(10) assert len(tasks) == 6 tasks = await scheduler.get_running_tasks() assert len(tasks) == 2 # They end... await asyncio.sleep(20) scheduler.max_running_tasks = 10 await asyncio.sleep(11) tasks = await scheduler.get_running_tasks() assert len(tasks) == 10 await self.stop_scheduler(scheduler)
async def test_timed_schedule(self): """Testing a timed schedule using a specific timestamp (in seconds) :assert: Number of running tasks The values declared at for timestamp """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() timed_schedule = TimedSchedule() # Set current timestamp to be: Tuesday August 8 2017 8:00:00 AM PDT now = 1502204400 scheduler.current_time = now timed_schedule.name = 'timed' timed_schedule.process_name = 'sleep10' timed_schedule.day = 2 timed_schedule.time = datetime.time(hour=8) timed_schedule.repeat = datetime.timedelta(seconds=0) timed_schedule.enabled = True # Set env timezone os.environ["TZ"] = "PST8PDT" time.tzset() await scheduler.save_schedule(timed_schedule) await asyncio.sleep(1) tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 timed_schedule = await scheduler.get_schedule( uuid.UUID(str(timed_schedule.schedule_id))) # Assert timed_schedule values assert timed_schedule.time.hour == 8 assert timed_schedule.time.minute == 0 assert timed_schedule.time.second == 0 assert timed_schedule.day == 2 # Reset timezone del os.environ["TZ"] time.tzset() await self.stop_scheduler(scheduler)
async def test_get_schedule(self): """Schedule gets retrieved :assert: Schedule is retrieved by id """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # Declare schedule interval_schedule = IntervalSchedule() interval_schedule.name = 'get_schedule_test' interval_schedule.process_name = "sleep30" interval_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(interval_schedule) # Get schedule schedules = await scheduler.get_schedules() assert len(schedules) == 1 # Assert the number of schedules await scheduler.get_schedule( interval_schedule.schedule_id ) # Get the schedule by schedule process ID # Assert that schedule is retrieved by ID try: await scheduler.get_schedule(uuid.uuid4()) assert False except ScheduleNotFoundError: pass await self.stop_scheduler(scheduler)
async def test_manual_schedule(self): """Test manually ran scheduled processes :assert: The number of running processes """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # Declare manual interval schedule manual_schedule = ManualSchedule() manual_schedule.name = 'manual task' manual_schedule.process_name = 'sleep10' manual_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(manual_schedule) manual_schedule = await scheduler.get_schedule( manual_schedule.schedule_id) await scheduler.queue_task(manual_schedule.schedule_id ) # Added a task to the _scheduler queue await asyncio.sleep(5) tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 await self.stop_scheduler(scheduler)
async def test_interval_none_repeat(self): """Tests an interval schedule where repeat is None :assert: A task starts immediately and doesn't repeat """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # assert that the schedule type is interval interval_schedule = IntervalSchedule() assert interval_schedule.schedule_type == Schedule.Type.INTERVAL interval_schedule.name = 'sleep10' interval_schedule.process_name = "sleep10" interval_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(interval_schedule) await asyncio.sleep(1) # Assert only 1 task is running tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 await asyncio.sleep(12) # Assert only 1 task is running tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 await self.stop_scheduler(scheduler)
async def test_modify_schedule_type(self): """Test modifying the type of a schedule """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() interval_schedule = IntervalSchedule() interval_schedule.name = 'sleep10' interval_schedule.process_name = 'sleep10' interval_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(interval_schedule) manual_schedule = ManualSchedule() manual_schedule.schedule_id = interval_schedule.schedule_id manual_schedule.name = 'manual' manual_schedule.process_name = 'sleep10' manual_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(manual_schedule) # Assert: only 1 task is running schedule = await scheduler.get_schedule(manual_schedule.schedule_id) assert isinstance(schedule, ManualSchedule) await self.stop_scheduler(scheduler)
async def test_delete(self): """Test that a scheduled process gets removed :assert: scheduled task gets removed """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # Set schedule to be interval based interval_schedule = IntervalSchedule() interval_schedule.name = 'deletetest' interval_schedule.process_name = "sleep1" interval_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(interval_schedule) await asyncio.sleep(5) # Delete a scheduled task await scheduler.delete_schedule(interval_schedule.schedule_id) # Assert that process was deleted try: await scheduler.delete_schedule(interval_schedule.schedule_id) assert False except ScheduleNotFoundError: pass await self.stop_scheduler(scheduler)
async def test_get_task(self): """Test tasks exists :assert: there exists a task """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() interval_schedule = IntervalSchedule() interval_schedule.name = 'get_task' interval_schedule.process_name = "sleep30" interval_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(interval_schedule) await asyncio.sleep(1) tasks = await scheduler.get_running_tasks( ) # retrieve list running tasks assert len(tasks) task = await scheduler.get_task(str(tasks[0].task_id)) assert task # assert there exists a task await self.stop_scheduler(scheduler)
async def test_startup_schedule(self): """Test startup of _scheduler :assert: the number of running tasks """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # Declare schedule startup, and execute startup_schedule = StartUpSchedule( ) # A scheduled process of the _scheduler startup_schedule.name = 'startup schedule' startup_schedule.process_name = 'sleep30' startup_schedule.repeat = datetime.timedelta( seconds=0) # set no repeat to startup startup_schedule.enabled = True await scheduler.save_schedule(startup_schedule) await asyncio.sleep(1) # Assert no tasks ar running tasks = await scheduler.get_running_tasks() assert len(tasks) == 0 await scheduler.get_schedule(startup_schedule.schedule_id ) # ID of the schedule startup await self.stop_scheduler(scheduler) scheduler = Scheduler() await scheduler.start() await asyncio.sleep(2) # Assert only 1 task is running tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 scheduler.max_running_tasks = 0 # set that no tasks would run await scheduler.cancel_task(tasks[0].task_id) await asyncio.sleep(10) # Assert no tasks are running tasks = await scheduler.get_running_tasks() assert len(tasks) == 0 scheduler.max_running_tasks = 1 await asyncio.sleep(2) # Assert a single task is running tasks = await scheduler.get_running_tasks() assert len(tasks) == 1 await self.stop_scheduler(scheduler)
async def test_disable_schedule(self): await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # Declare schedule interval_schedule = IntervalSchedule() interval_schedule.name = 'disable_schedule_test' interval_schedule.process_name = "sleep5" interval_schedule.repeat = datetime.timedelta(seconds=0) interval_schedule.enabled = True await scheduler.save_schedule(interval_schedule) # Get schedule schedules = await scheduler.get_schedules() assert len(schedules) == 1 # Assert the number of schedules assert schedules[0].enabled is True await asyncio.sleep(5) # assert there exists a task tasks = await scheduler.get_running_tasks( ) # retrieve list running tasks assert len(tasks) task = await scheduler.get_task(tasks[0].task_id) assert task # Disable Schedule retval, reason = await scheduler.disable_schedule( interval_schedule.schedule_id) assert retval # Confirm enabled changed schedules = await scheduler.get_schedules() assert len(schedules) == 1 # Assert the number of schedules assert schedules[0].enabled is False await self.stop_scheduler(scheduler)
async def test_stop(self): """Test that stop_scheduler actually works""" await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # Set schedule interval interval_schedule = IntervalSchedule() interval_schedule.exclusive = False interval_schedule.name = 'sleep1' interval_schedule.process_name = "sleep1" interval_schedule.repeat = datetime.timedelta( seconds=1) # Set frequency of await scheduler.save_schedule(interval_schedule ) # Save schedule updates await asyncio.sleep(10) await self.stop_scheduler(scheduler)
async def test_cancel(self): """Cancel a running process""" await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() interval_schedule = IntervalSchedule() interval_schedule.name = 'cancel_test' interval_schedule.process_name = 'sleep30' interval_schedule.repeat = datetime.timedelta(seconds=0) await scheduler.save_schedule(interval_schedule) await asyncio.sleep(5) tasks = await scheduler.get_running_tasks() await scheduler.cancel_task(tasks[0].task_id) # Cancel a running task await self.stop_scheduler(scheduler)
async def test_create_interval(self): """Test the creation of a new schedule interval :assert: The interval type of the schedule """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # assert that the schedule type is interval interval_schedule = IntervalSchedule() assert interval_schedule.schedule_type == Schedule.Type.INTERVAL interval_schedule.name = 'sleep10' interval_schedule.process_name = "sleep10" interval_schedule.repeat = datetime.timedelta(seconds=1) await scheduler.save_schedule(interval_schedule) await self.stop_scheduler(scheduler)
async def test_add_notification_service(self, client, payload): data = json.loads(payload) sch_id = '45876056-e04c-4cde-8a82-1d8dbbbe6d72' async def async_mock_get_schedule(): schedule = StartUpSchedule() schedule.schedule_id = sch_id return schedule @asyncio.coroutine def q_result(*arg): table = arg[0] _payload = json.loads(arg[1]) if table == 'schedules': if _payload['return'][0] == 'process_name': assert {"return": ["process_name"]} == _payload return { 'rows': [{ 'process_name': 'purge' }, { 'process_name': 'stats collector' }], 'count': 2 } else: assert { "return": ["schedule_name"], "where": { "column": "schedule_name", "condition": "=", "value": data['name'] } } == _payload return {'count': 0, 'rows': []} if table == 'scheduled_processes': assert { "return": ["name"], "where": { "column": "name", "condition": "=", "value": "notification_c" } } == _payload return {'count': 0, 'rows': []} expected_insert_resp = {'rows_affected': 1, "response": "inserted"} server.Server.scheduler = Scheduler(None, None) storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object( c_mgr, 'get_category_all_items', return_value=self.async_mock(None)) as patch_get_cat_info: with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object( storage_client_mock, 'insert_into_tbl', return_value=self.async_mock( expected_insert_resp)) as insert_table_patch: with patch.object(server.Server.scheduler, 'save_schedule', return_value=self.async_mock( "")) as patch_save_schedule: with patch.object( server.Server.scheduler, 'get_schedule_by_name', return_value=async_mock_get_schedule( )) as patch_get_schedule: resp = await client.post('/foglamp/service', data=payload) server.Server.scheduler = None assert 200 == resp.status result = await resp.text() json_response = json.loads(result) assert { 'id': sch_id, 'name': data['name'] } == json_response patch_get_schedule.assert_called_once_with( data['name']) patch_save_schedule.called_once_with() args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert { 'name': 'notification_c', 'script': '["services/notification_c"]' } == p patch_get_cat_info.assert_called_once_with( category_name=data['name'])
def setup_method(self): server.Server.scheduler = Scheduler(None, None)
async def test_add_service(self, client, payload): data = json.loads(payload) async def async_mock_get_schedule(): schedule = StartUpSchedule() schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b' return schedule @asyncio.coroutine def q_result(*arg): table = arg[0] _payload = arg[1] if table == 'scheduled_processes': assert { 'return': ['name'], 'where': { 'column': 'name', 'condition': '=', 'value': 'south_c' } } == json.loads(_payload) return {'count': 0, 'rows': []} if table == 'schedules': assert { 'return': ['schedule_name'], 'where': { 'column': 'schedule_name', 'condition': '=', 'value': 'furnace4' } } == json.loads(_payload) return {'count': 0, 'rows': []} expected_insert_resp = {'rows_affected': 1, "response": "inserted"} mock_plugin_info = { 'name': "furnace4", 'version': "1.1", 'type': "south", 'interface': "1.0", 'mode': "async", 'config': { 'plugin': { 'description': "DHT11 plugin", 'type': 'string', 'default': 'dht11' } } } mock = MagicMock() attrs = {"plugin_info.side_effect": [mock_plugin_info]} mock.configure_mock(**attrs) server.Server.scheduler = Scheduler(None, None) storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch('builtins.__import__', return_value=mock): with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock( None)) as patch_get_cat_info: with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock(expected_insert_resp)) \ as insert_table_patch: with patch.object(c_mgr, 'create_category', return_value=self.async_mock( None)) as patch_create_cat: with patch.object(c_mgr, 'create_child_category', return_value=self.async_mock(None)) \ as patch_create_child_cat: with patch.object( server.Server.scheduler, 'save_schedule', return_value=self.async_mock( "")) as patch_save_schedule: with patch.object( server.Server.scheduler, 'get_schedule_by_name', return_value= async_mock_get_schedule( )) as patch_get_schedule: resp = await client.post( '/foglamp/service', data=payload) server.Server.scheduler = None assert 200 == resp.status result = await resp.text() json_response = json.loads(result) assert { 'id': '2129cc95-c841-441a-ad39-6469a87dbc8b', 'name': 'furnace4' } == json_response patch_get_schedule.assert_called_once_with( data['name']) patch_save_schedule.called_once_with() patch_create_child_cat.assert_called_once_with( 'South', ['furnace4']) assert 2 == patch_create_cat.call_count patch_create_cat.assert_called_with( 'South', {}, 'South microservices', True) args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert { 'name': 'south_c', 'script': '["services/south_c"]' } == p patch_get_cat_info.assert_called_once_with( category_name='furnace4')
sqlalchemy.Column('process_name', sqlalchemy.types.VARCHAR(20), default=''), sqlalchemy.Column('state', sqlalchemy.types.INT), sqlalchemy.Column('start_time', sqlalchemy.types.TIMESTAMP), sqlalchemy.Column('end_time', sqlalchemy.types.TIMESTAMP), sqlalchemy.Column('pid', sqlalchemy.types.INT), sqlalchemy.Column('exit_code', sqlalchemy.types.INT), sqlalchemy.Column('reason', sqlalchemy.types.VARCHAR(255))) # TODO: FOGL-1017 :To run this test, FOGLAMP_ENV=TEST is only used by scheduler # 1) Execute this command: FOGLAMP_ENV=TEST pytest -s -vv tests/integration/foglamp/services/core/test_scheduler_get_tasks.py # TODO: How to eliminate manual intervention as below when tests will run unattended at CI? _address = pytest.test_env.address _m_port = pytest.test_env.core_mgmt_port scheduler = Scheduler(_address, _m_port) @pytest.allure.feature("integration") @pytest.allure.story("scheduler get_tasks") class TestSchedulerGetTasks: _engine = None # type: aiopg.sa.Engine # TODO: This test will not work if our storage engine is not Postgres. OK for today but long term we need to # approach this differently. We could simply use the storage layer to insert the test data. @classmethod async def _get_connection_pool(cls) -> aiopg.sa.Engine: """Returns a database connection pool object""" if cls._engine is None: cls._engine = await aiopg.sa.create_engine(_CONNECTION_STRING) return cls._engine
async def _start_scheduler(cls): """Starts the scheduler""" _logger.info("start scheduler") cls.scheduler = Scheduler(cls._host, cls.core_management_port) await cls.scheduler.start()
async def test_add_service(self, client): async def async_mock(return_value): return return_value async def async_mock_get_schedule(): schedule = StartUpSchedule() schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b' return schedule def q_result(*arg): table = arg[0] payload = arg[1] if table == 'scheduled_processes': assert { 'return': ['name'], 'where': { 'column': 'name', 'condition': '=', 'value': 'furnace4' } } == json.loads(payload) return {'count': 0, 'rows': []} if table == 'schedules': assert { 'return': ['schedule_name'], 'where': { 'column': 'schedule_name', 'condition': '=', 'value': 'furnace4' } } == json.loads(payload) return {'count': 0, 'rows': []} async def async_mock_insert(): expected = {'rows_affected': 1, "response": "inserted"} return expected server.Server.scheduler = Scheduler(None, None) data = {"name": "furnace4", "type": "south", "plugin": "dht11"} description = '{} service configuration'.format(data['name']) storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) val = { 'plugin': { 'default': data['plugin'], 'description': 'Python module name of the plugin to load', 'type': 'string' } } with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object(storage_client_mock, 'insert_into_tbl', return_value=async_mock_insert() ) as insert_table_patch: with patch.object( c_mgr, 'create_category', return_value=async_mock(None)) as patch_create_cat: with patch.object(server.Server.scheduler, 'save_schedule', return_value=async_mock( "")) as patch_save_schedule: with patch.object( server.Server.scheduler, 'get_schedule_by_name', return_value=async_mock_get_schedule( )) as patch_get_schedule: resp = await client.post('/foglamp/service', data=json.dumps(data)) server.Server.scheduler = None assert 200 == resp.status result = await resp.text() json_response = json.loads(result) assert { 'id': '2129cc95-c841-441a-ad39-6469a87dbc8b', 'name': 'furnace4' } == json_response patch_get_schedule.assert_called_once_with( data['name']) patch_save_schedule.called_once_with() patch_create_cat.assert_called_once_with( category_name=data['name'], category_description=description, category_value=val, keep_original_items=True) args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert { 'name': 'furnace4', 'script': '["services/south"]' } == p
async def test_add_task_with_config(self, client): async def async_mock_get_schedule(): schedule = TimedSchedule() schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b' return schedule @asyncio.coroutine def q_result(*arg): table = arg[0] payload = arg[1] if table == 'scheduled_processes': assert { 'return': ['name'], 'where': { 'column': 'name', 'condition': '=', 'value': 'north' } } == json.loads(payload) return {'count': 0, 'rows': []} if table == 'schedules': assert { 'return': ['schedule_name'], 'where': { 'column': 'schedule_name', 'condition': '=', 'value': 'north bound' } } == json.loads(payload) return {'count': 0, 'rows': []} if table == 'tasks': return {'count': 0, 'rows': []} expected_insert_resp = {'rows_affected': 1, "response": "inserted"} mock_plugin_info = { 'name': "PI server", 'version': "1.1", 'type': "north", 'interface': "1.0", 'config': { 'plugin': { 'description': "North PI plugin", 'type': 'string', 'default': 'omf' }, 'producerToken': { 'description': 'Producer token for this FogLAMP stream', 'type': 'string', 'default': 'pi_server_north_0001', 'order': '2' } } } mock = MagicMock() attrs = {"plugin_info.side_effect": [mock_plugin_info]} mock.configure_mock(**attrs) server.Server.scheduler = Scheduler(None, None) data = { "name": "north bound", "plugin": "omf", "type": "north", "schedule_type": 3, "schedule_day": 0, "schedule_time": 0, "schedule_repeat": 30, "schedule_enabled": True, "config": { "producerToken": { "value": "uid=180905062754237&sig=kx5l+" } } } storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch('builtins.__import__', return_value=mock): with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock( None)) as patch_get_cat_info: with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock(expected_insert_resp)) \ as insert_table_patch: with patch.object(c_mgr, 'create_category', return_value=self.async_mock( None)) as patch_create_cat: with patch.object(c_mgr, 'create_child_category', return_value=self.async_mock(None)) \ as patch_create_child_cat: with patch.object( c_mgr, 'set_category_item_value_entry', return_value=self.async_mock( None)) as patch_set_entry: with patch.object( server.Server.scheduler, 'save_schedule', return_value=self.async_mock( "") ) as patch_save_schedule: with patch.object( server.Server.scheduler, 'get_schedule_by_name', return_value= async_mock_get_schedule( )) as patch_get_schedule: resp = await client.post( '/foglamp/scheduled/task', data=json.dumps(data)) server.Server.scheduler = None assert 200 == resp.status result = await resp.text() json_response = json.loads( result) assert { 'id': '2129cc95-c841-441a-ad39-6469a87dbc8b', 'name': 'north bound' } == json_response patch_get_schedule.assert_called_once_with( data['name']) patch_save_schedule.called_once_with() patch_set_entry.assert_called_once_with( data['name'], 'producerToken', 'uid=180905062754237&sig=kx5l+') patch_create_child_cat.assert_called_once_with( 'North', ['north bound']) assert 2 == patch_create_cat.call_count patch_create_cat.assert_called_with( 'North', {}, 'North tasks', True) args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert p['name'] == 'north' assert p['script'] == '["tasks/north"]' patch_get_cat_info.assert_called_once_with( category_name=data['name'])
async def test_dupe_notification_service_schedule(self, client): payload = '{"name": "NotificationServer", "type": "notification"}' data = json.loads(payload) @asyncio.coroutine def q_result(*arg): table = arg[0] _payload = json.loads(arg[1]) if table == 'schedules': if _payload['return'][0] == 'process_name': assert {"return": ["process_name"]} == _payload return { 'rows': [{ 'process_name': 'stats collector' }, { 'process_name': 'notification_c' }], 'count': 2 } else: assert { "return": ["schedule_name"], "where": { "column": "schedule_name", "condition": "=", "value": data['name'] } } == _payload return {'count': 0, 'rows': []} if table == 'scheduled_processes': assert { "return": ["name"], "where": { "column": "name", "condition": "=", "value": "notification_c" } } == _payload return {'count': 0, 'rows': []} expected_insert_resp = {'rows_affected': 1, "response": "inserted"} server.Server.scheduler = Scheduler(None, None) storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object( c_mgr, 'get_category_all_items', return_value=self.async_mock(None)) as patch_get_cat_info: with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object( storage_client_mock, 'insert_into_tbl', return_value=self.async_mock( expected_insert_resp)) as insert_table_patch: resp = await client.post('/foglamp/service', data=payload) server.Server.scheduler = None assert 400 == resp.status assert 'A Notification service schedule already exists.' == resp.reason args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert { 'name': 'notification_c', 'script': '["services/notification_c"]' } == p patch_get_cat_info.assert_called_once_with( category_name=data['name'])
async def test_add_task(self, client): async def async_mock_get_schedule(): schedule = TimedSchedule() schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b' return schedule @asyncio.coroutine def q_result(*arg): table = arg[0] payload = arg[1] if table == 'scheduled_processes': assert { 'return': ['name'], 'where': { 'column': 'name', 'condition': '=', 'value': 'north' } } == json.loads(payload) return {'count': 0, 'rows': []} if table == 'schedules': assert { 'return': ['schedule_name'], 'where': { 'column': 'schedule_name', 'condition': '=', 'value': 'north bound' } } == json.loads(payload) return {'count': 0, 'rows': []} if table == 'tasks': return {'count': 0, 'rows': []} expected_insert_resp = {'rows_affected': 1, "response": "inserted"} mock_plugin_info = { 'name': "north bound", 'version': "1.1", 'type': "north", 'interface': "1.0", 'config': { 'plugin': { 'description': "North OMF plugin", 'type': 'string', 'default': 'omf' } } } server.Server.scheduler = Scheduler(None, None) data = { "name": "north bound", "plugin": "omf", "type": "north", "schedule_type": 3, "schedule_day": 0, "schedule_time": 0, "schedule_repeat": 30, "schedule_enabled": True } storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(common, 'load_and_fetch_python_plugin_info', side_effect=[mock_plugin_info]): with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock( None)) as patch_get_cat_info: with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock(expected_insert_resp)) \ as insert_table_patch: with patch.object(c_mgr, 'create_category', return_value=self.async_mock( None)) as patch_create_cat: with patch.object(c_mgr, 'create_child_category', return_value=self.async_mock(None)) \ as patch_create_child_cat: with patch.object( server.Server.scheduler, 'save_schedule', return_value=self.async_mock( "")) as patch_save_schedule: with patch.object( server.Server.scheduler, 'get_schedule_by_name', return_value= async_mock_get_schedule( )) as patch_get_schedule: resp = await client.post( '/foglamp/scheduled/task', data=json.dumps(data)) server.Server.scheduler = None assert 200 == resp.status result = await resp.text() json_response = json.loads(result) assert { 'id': '2129cc95-c841-441a-ad39-6469a87dbc8b', 'name': 'north bound' } == json_response patch_get_schedule.assert_called_once_with( data['name']) patch_save_schedule.called_once_with() patch_create_child_cat.assert_called_once_with( 'North', ['north bound']) calls = [ call(category_description='North OMF plugin', category_name='north bound', category_value={ 'plugin': { 'description': 'North OMF plugin', 'default': 'omf', 'type': 'string' } }, keep_original_items=True), call('North', {}, 'North tasks', True) ] patch_create_cat.assert_has_calls(calls) args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert p['name'] == 'north' assert p['script'] == '["tasks/north"]' patch_get_cat_info.assert_called_once_with( category_name=data['name'])
async def test_add_service_with_config(self, client): payload = '{"name": "Sine", "type": "south", "plugin": "sinusoid", "enabled": "false",' \ ' "config": {"dataPointsPerSec": {"value": "10"}}}' data = json.loads(payload) async def async_mock_get_schedule(): schedule = StartUpSchedule() schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b' return schedule @asyncio.coroutine def q_result(*arg): table = arg[0] _payload = arg[1] if table == 'scheduled_processes': assert { 'return': ['name'], 'where': { 'column': 'name', 'condition': '=', 'value': 'south_c' } } == json.loads(_payload) return {'count': 0, 'rows': []} if table == 'schedules': assert { 'return': ['schedule_name'], 'where': { 'column': 'schedule_name', 'condition': '=', 'value': data['name'] } } == json.loads(_payload) return {'count': 0, 'rows': []} expected_insert_resp = {'rows_affected': 1, "response": "inserted"} mock_plugin_info = { 'name': data['name'], 'version': "1.1", 'type': "south", 'interface': "1.0", 'mode': "async", 'config': { 'plugin': { 'description': "Sinusoid Plugin", 'type': 'string', 'default': 'sinusoid' }, 'dataPointsPerSec': { 'description': 'Data points per second', 'type': 'integer', 'default': '1', 'order': '2' } } } server.Server.scheduler = Scheduler(None, None) storage_client_mock = MagicMock(StorageClientAsync) c_mgr = ConfigurationManager(storage_client_mock) with patch.object(common, 'load_and_fetch_python_plugin_info', side_effect=[mock_plugin_info]): with patch.object(connect, 'get_storage_async', return_value=storage_client_mock): with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock( None)) as patch_get_cat_info: with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result): with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock( expected_insert_resp) ) as insert_table_patch: with patch.object(c_mgr, 'create_category', return_value=self.async_mock( None)) as patch_create_cat: with patch.object( c_mgr, 'create_child_category', return_value=self.async_mock( None)) as patch_create_child_cat: with patch.object( c_mgr, 'set_category_item_value_entry', return_value=self.async_mock( None)) as patch_set_entry: with patch.object( server.Server.scheduler, 'save_schedule', return_value=self.async_mock( "") ) as patch_save_schedule: with patch.object( server.Server.scheduler, 'get_schedule_by_name', return_value= async_mock_get_schedule( )) as patch_get_schedule: resp = await client.post( '/foglamp/service', data=payload) server.Server.scheduler = None assert 200 == resp.status result = await resp.text() json_response = json.loads( result) assert { 'id': '2129cc95-c841-441a-ad39-6469a87dbc8b', 'name': data['name'] } == json_response patch_get_schedule.assert_called_once_with( data['name']) patch_save_schedule.called_once_with() patch_set_entry.assert_called_once_with( data['name'], 'dataPointsPerSec', '10') patch_create_child_cat.assert_called_once_with( 'South', ['Sine']) assert 2 == patch_create_cat.call_count patch_create_cat.assert_called_with( 'South', {}, 'South microservices', True) args, kwargs = insert_table_patch.call_args assert 'scheduled_processes' == args[0] p = json.loads(args[1]) assert { 'name': 'south_c', 'script': '["services/south_c"]' } == p patch_get_cat_info.assert_called_once_with( category_name=data['name'])
async def test_get_tasks(self): """Get list of tasks :assert: Number of running tasks The state of tasks the start time of a given task """ await self.populate_test_data( ) # Populate data in foglamp.scheduled_processes scheduler = Scheduler(_address, _m_port) await scheduler.start() # declare _scheduler task interval_schedule = IntervalSchedule() interval_schedule.name = 'get_tasks' interval_schedule.process_name = "sleep5" interval_schedule.repeat = datetime.timedelta(seconds=1) interval_schedule.exclusive = False await scheduler.save_schedule(interval_schedule) await asyncio.sleep(15) # Assert running tasks tasks = await scheduler.get_tasks( where=["state", "=", int(Task.State.INTERRUPTED)]) assert not tasks tasks = await scheduler.get_tasks(where=["end_time", "=", 'NULL']) assert tasks tasks = await scheduler.get_tasks(limit=50) states = [int(task.state) for task in tasks] assert len(tasks) > 1 assert int(Task.State.RUNNING) in states assert int(Task.State.COMPLETE) in states tasks = await scheduler.get_tasks(1) assert len(tasks) == 1 tasks = await scheduler.get_tasks( where=["state", "=", int(Task.State.RUNNING)], sort=[["state", "desc"]], offset=50) assert not tasks tasks = await scheduler.get_tasks( where=["state", "=", int(Task.State.RUNNING)], sort=[["state", "desc"], ["start_time", "asc"]]) assert tasks tasks = await scheduler.get_tasks(or_where_list=[["state", "=", int(Task.State.RUNNING)], \ ["state", "=", int(Task.State.RUNNING)]]) assert tasks tasks = await scheduler.get_tasks(and_where_list=[["state", "=", int(Task.State.RUNNING)], \ ["state", "=", int(Task.State.RUNNING)]]) assert tasks await self.stop_scheduler(scheduler)
async def get_statistics_history(request): """ Args: request: Returns: a list of general set of statistics :Example: curl -X GET http://localhost:8081/foglamp/statistics/history?limit=1 """ storage_client = connect.get_storage() # To find the interval in secs from stats collector schedule scheduler_payload = PayloadBuilder().SELECT("schedule_interval").WHERE( ['process_name', '=', 'stats collector']).payload() result = storage_client.query_tbl_with_payload('schedules', scheduler_payload) if len(result['rows']) > 0: scheduler = Scheduler() interval_days, interval_dt = scheduler.extract_day_time_from_interval( result['rows'][0]['schedule_interval']) interval = datetime.timedelta(days=interval_days, hours=interval_dt.hour, minutes=interval_dt.minute, seconds=interval_dt.second) interval_in_secs = interval.total_seconds() else: raise web.HTTPNotFound(reason="No stats collector schedule found") stats_history_chain_payload = PayloadBuilder().SELECT(("history_ts", "key", "value"))\ .ALIAS("return", ("history_ts", 'history_ts')).FORMAT("return", ("history_ts", "YYYY-MM-DD HH24:MI:SS.MS"))\ .ORDER_BY(['history_ts', 'desc']).chain_payload() if 'limit' in request.query and request.query['limit'] != '': try: limit = int(request.query['limit']) if limit < 0: raise ValueError # FIXME: Hack straight away multiply the LIMIT by the group count # i.e. if there are 8 records per distinct (stats_key), and limit supplied is 2 # then internally, actual LIMIT = 2*8 # TODO: FOGL-663 Need support for "subquery" from storage service # Remove python side handling date_trunc and use # SELECT date_trunc('second', history_ts::timestamptz)::varchar as history_ts count_payload = PayloadBuilder().AGGREGATE(["count", "*"]).payload() result = storage_client.query_tbl_with_payload( "statistics", count_payload) key_count = result['rows'][0]['count_*'] stats_history_chain_payload = PayloadBuilder( stats_history_chain_payload).LIMIT(limit * key_count).chain_payload() except ValueError: raise web.HTTPBadRequest(reason="Limit must be a positive integer") stats_history_payload = PayloadBuilder( stats_history_chain_payload).payload() result_from_storage = storage_client.query_tbl_with_payload( 'statistics_history', stats_history_payload) group_dict = [] for row in result_from_storage['rows']: new_dict = {'history_ts': row['history_ts'], row['key']: row['value']} group_dict.append(new_dict) results = [] temp_dict = {} previous_ts = None for row in group_dict: # first time or when history_ts changes if previous_ts is None or previous_ts != row['history_ts']: if previous_ts is not None: results.append(temp_dict) previous_ts = row['history_ts'] temp_dict = {'history_ts': previous_ts} # Append statistics key to temp dict for key, value in row.items(): temp_dict.update({key: value}) # Append the last set of records which do not get appended above results.append(temp_dict) return web.json_response({ "interval": interval_in_secs, 'statistics': results })
async def test_disable_schedule_wrong_schedule_id(self): with pytest.raises(ScheduleNotFoundError) as excinfo: scheduler = Scheduler(_address, _m_port) await scheduler.start() random_schedule_id = uuid.uuid4() await scheduler.disable_schedule(random_schedule_id)