async def test_add_notification_service(self, client, payload):
data = json.loads(payload)
sch_id = '45876056-e04c-4cde-8a82-1d8dbbbe6d72'
async def async_mock_get_schedule():
schedule = StartUpSchedule()
schedule.schedule_id = sch_id
return schedule
@asyncio.coroutine
def q_result(*arg):
table = arg[0]
_payload = json.loads(arg[1])
if table == 'schedules':
if _payload['return'][0] == 'process_name':
assert {"return": ["process_name"]} == _payload
return {'rows': [{'process_name': 'purge'}, {'process_name': 'stats collector'}], 'count': 2}
else:
assert {"return": ["schedule_name"], "where": {"column": "schedule_name", "condition": "=",
"value": data['name']}} == _payload
return {'count': 0, 'rows': []}
if table == 'scheduled_processes':
assert {"return": ["name"], "where": {"column": "name", "condition": "=",
"value": "notification_c"}} == _payload
return {'count': 0, 'rows': []}
expected_insert_resp = {'rows_affected': 1, "response": "inserted"}
server.Server.scheduler = Scheduler(None, None)
storage_client_mock = MagicMock(StorageClientAsync)
c_mgr = ConfigurationManager(storage_client_mock)
with patch.object(connect, 'get_storage_async', return_value=storage_client_mock):
with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock(None)) as patch_get_cat_info:
with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result):
with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock(expected_insert_resp)) as insert_table_patch:
with patch.object(server.Server.scheduler, 'save_schedule', return_value=self.async_mock("")) as patch_save_schedule:
with patch.object(server.Server.scheduler, 'get_schedule_by_name', return_value=async_mock_get_schedule()) as patch_get_schedule:
resp = await client.post('/fledge/service', data=payload)
server.Server.scheduler = None
assert 200 == resp.status
result = await resp.text()
json_response = json.loads(result)
assert {'id': sch_id, 'name': data['name']} == json_response
patch_get_schedule.assert_called_once_with(data['name'])
patch_save_schedule.called_once_with()
args, kwargs = insert_table_patch.call_args
assert 'scheduled_processes' == args[0]
p = json.loads(args[1])
assert {'name': 'notification_c', 'script': '["services/notification_c"]'} == p
patch_get_cat_info.assert_called_once_with(category_name=data['name'])
async def test_dupe_notification_service_schedule(self, client):
payload = '{"name": "NotificationServer", "type": "notification"}'
data = json.loads(payload)
@asyncio.coroutine
def q_result(*arg):
table = arg[0]
_payload = json.loads(arg[1])
if table == 'schedules':
if _payload['return'][0] == 'process_name':
assert {"return": ["process_name"]} == _payload
return {'rows': [{'process_name': 'stats collector'}, {'process_name': 'notification_c'}], 'count': 2}
else:
assert {"return": ["schedule_name"], "where": {"column": "schedule_name", "condition": "=",
"value": data['name']}} == _payload
return {'count': 0, 'rows': []}
if table == 'scheduled_processes':
assert {"return": ["name"], "where": {"column": "name", "condition": "=",
"value": "notification_c"}} == _payload
return {'count': 0, 'rows': []}
expected_insert_resp = {'rows_affected': 1, "response": "inserted"}
server.Server.scheduler = Scheduler(None, None)
storage_client_mock = MagicMock(StorageClientAsync)
c_mgr = ConfigurationManager(storage_client_mock)
with patch.object(connect, 'get_storage_async', return_value=storage_client_mock):
with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock(None)) as patch_get_cat_info:
with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result):
with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock(expected_insert_resp)) as insert_table_patch:
resp = await client.post('/fledge/service', data=payload)
server.Server.scheduler = None
assert 400 == resp.status
assert 'A Notification service schedule already exists.' == resp.reason
args, kwargs = insert_table_patch.call_args
assert 'scheduled_processes' == args[0]
p = json.loads(args[1])
assert {'name': 'notification_c', 'script': '["services/notification_c"]'} == p
patch_get_cat_info.assert_called_once_with(category_name=data['name'])
def setup_method(self):
server.Server.scheduler = Scheduler(None, None)
async def get_statistics_history(request):
"""
Args:
request:
Returns:
a list of general set of statistics
:Example:
curl -X GET http://localhost:8081/fledge/statistics/history?limit=1
curl -X GET http://localhost:8081/fledge/statistics/history?key=READINGS
curl -X GET http://localhost:8081/fledge/statistics/history?key=READINGS,PURGED,UNSENT&minutes=60
"""
storage_client = connect.get_storage_async()
# To find the interval in secs from stats collector schedule
scheduler_payload = PayloadBuilder().SELECT("schedule_interval").WHERE(
['process_name', '=', 'stats collector']).payload()
result = await storage_client.query_tbl_with_payload(
'schedules', scheduler_payload)
if len(result['rows']) > 0:
scheduler = Scheduler()
interval_days, interval_dt = scheduler.extract_day_time_from_interval(
result['rows'][0]['schedule_interval'])
interval = datetime.timedelta(days=interval_days,
hours=interval_dt.hour,
minutes=interval_dt.minute,
seconds=interval_dt.second)
interval_in_secs = interval.total_seconds()
else:
raise web.HTTPNotFound(reason="No stats collector schedule found")
stats_history_chain_payload = PayloadBuilder().SELECT(("history_ts", "key", "value"))\
.ALIAS("return", ("history_ts", 'history_ts')).FORMAT("return", ("history_ts", "YYYY-MM-DD HH24:MI:SS.MS"))\
.ORDER_BY(['history_ts', 'desc']).WHERE(['1', '=', 1]).chain_payload()
if 'key' in request.query:
key = request.query['key']
split_list = key.split(',')
stats_history_chain_payload = PayloadBuilder(
stats_history_chain_payload).AND_WHERE(['key', '=', split_list[0]
]).chain_payload()
del split_list[0]
for i in split_list:
stats_history_chain_payload = PayloadBuilder(
stats_history_chain_payload).OR_WHERE(['key', '=',
i]).chain_payload()
try:
# get time based graphs for statistics history
val = 0
if 'minutes' in request.query and request.query['minutes'] != '':
val = int(request.query['minutes']) * 60
elif 'hours' in request.query and request.query['hours'] != '':
val = int(request.query['hours']) * 60 * 60
elif 'days' in request.query and request.query['days'] != '':
val = int(request.query['days']) * 24 * 60 * 60
if val < 0:
raise ValueError
elif val > 0:
stats_history_chain_payload = PayloadBuilder(
stats_history_chain_payload).AND_WHERE(
['history_ts', 'newer', val]).chain_payload()
except ValueError:
raise web.HTTPBadRequest(reason="Time unit must be a positive integer")
if 'limit' in request.query and request.query['limit'] != '':
try:
limit = int(request.query['limit'])
if limit < 0:
raise ValueError
if 'key' in request.query:
limit_count = limit
else:
# FIXME: Hack straight away multiply the LIMIT by the group count
# i.e. if there are 8 records per distinct (stats_key), and limit supplied is 2
# then internally, actual LIMIT = 2*8
# TODO: FOGL-663 Need support for "subquery" from storage service
# Remove python side handling date_trunc and use
# SELECT date_trunc('second', history_ts::timestamptz)::varchar as history_ts
count_payload = PayloadBuilder().AGGREGATE(["count",
"*"]).payload()
result = await storage_client.query_tbl_with_payload(
"statistics", count_payload)
key_count = result['rows'][0]['count_*']
limit_count = limit * key_count
stats_history_chain_payload = PayloadBuilder(
stats_history_chain_payload).LIMIT(
limit_count).chain_payload()
except ValueError:
raise web.HTTPBadRequest(reason="Limit must be a positive integer")
stats_history_payload = PayloadBuilder(
stats_history_chain_payload).payload()
result_from_storage = await storage_client.query_tbl_with_payload(
'statistics_history', stats_history_payload)
group_dict = []
for row in result_from_storage['rows']:
new_dict = {'history_ts': row['history_ts'], row['key']: row['value']}
group_dict.append(new_dict)
results = []
temp_dict = {}
previous_ts = None
for row in group_dict:
# first time or when history_ts changes
if previous_ts is None or previous_ts != row['history_ts']:
if previous_ts is not None:
results.append(temp_dict)
previous_ts = row['history_ts']
temp_dict = {'history_ts': previous_ts}
# Append statistics key to temp dict
for key, value in row.items():
temp_dict.update({key: value})
# Append the last set of records which do not get appended above
results.append(temp_dict)
return web.json_response({
"interval": interval_in_secs,
'statistics': results
})
async def get_statistics_rate(request: web.Request) -> web.Response:
"""
Args:
request:
Returns:
A JSON document with the rates for each of the statistics
:Example:
curl -X GET http://localhost:8081/fledge/statistics/rate?periods=1,5,15&statistics=SINUSOID,FASTSINUSOID,READINGS
Implementation:
Calculation via: (sum(value) / count(value)) * 60 / (<statistic history interval>)
Queries for above example:
select key, 4 * (sum(value) / count(value)) from statistics_history where history_ts >= datetime('now', '-1 Minute') and key in ("SINUSOID", "FASTSINUSOID", "READINGS" ) group by key;
select key, 4 * (sum(value) / count(value)) from statistics_history where history_ts >= datetime('now', '-5 Minute') and key in ("SINUSOID", "FASTSINUSOID", "READINGS" ) group by key;
select key, 4 * (sum(value) / count(value)) from statistics_history where history_ts >= datetime('now', '-15 Minute') and key in ("SINUSOID", "FASTSINUSOID", "READINGS" ) group by key;
"""
params = request.query
if 'periods' not in params:
raise web.HTTPBadRequest(
reason="periods request parameter is required")
if 'statistics' not in params:
raise web.HTTPBadRequest(
reason="statistics request parameter is required")
if params['periods'] == '':
raise web.HTTPBadRequest(
reason=
"periods cannot be an empty. Also comma separated list of values required "
"in case of multiple periods of time")
if params['statistics'] == '':
raise web.HTTPBadRequest(
reason=
"statistics cannot be an empty. Also comma separated list of statistics values "
"required in case of multiple assets")
periods = params['periods']
period_split_list = list(filter(None, periods.split(',')))
if not all(p.isdigit() for p in period_split_list):
raise web.HTTPBadRequest(reason="periods should contain numbers")
# 1 week = 10080 mins
if any(int(p) > 10800 for p in period_split_list):
raise web.HTTPBadRequest(
reason="The maximum allowed value for a period is 10080 minutes")
stats = params['statistics']
stat_split_list = list(filter(None, [x.upper() for x in stats.split(',')]))
storage_client = connect.get_storage_async()
# To find the interval in secs from stats collector schedule
scheduler_payload = PayloadBuilder().SELECT("schedule_interval").WHERE(
['process_name', '=', 'stats collector']).payload()
result = await storage_client.query_tbl_with_payload(
'schedules', scheduler_payload)
if len(result['rows']) > 0:
scheduler = Scheduler()
interval_days, interval_dt = scheduler.extract_day_time_from_interval(
result['rows'][0]['schedule_interval'])
interval_in_secs = datetime.timedelta(
days=interval_days,
hours=interval_dt.hour,
minutes=interval_dt.minute,
seconds=interval_dt.second).total_seconds()
else:
raise web.HTTPNotFound(reason="No stats collector schedule found")
ts = datetime.datetime.now().timestamp()
resp = []
for x, y in [(x, y) for x in period_split_list for y in stat_split_list]:
time_diff = ts - int(x)
# TODO: FOGL-4102
# For example:
# time_diff = 1590066814.037321
# ERROR: PostgreSQL storage plugin raising error: ERROR: invalid input syntax for type timestamp with time zone: "1590066814.037321"
# "where": {"column": "history_ts", "condition": ">=", "value": "1590066814.037321"} - Payload works with sqlite engine BUT not with postgres
# To overcome above problem on postgres - I have used "dt = 2020-05-21 13:13:34" - but I see some deviations in results for both engines when we use datetime format
_payload = PayloadBuilder().SELECT("key").AGGREGATE([
"sum", "value"
]).AGGREGATE(["count",
"value"]).WHERE(['history_ts', '>=',
str(time_diff)
]).AND_WHERE(['key', '=',
y]).chain_payload()
stats_rate_payload = PayloadBuilder(_payload).GROUP_BY("key").payload()
result = await storage_client.query_tbl_with_payload(
"statistics_history", stats_rate_payload)
temp_dict = {y: {x: 0}}
if result['rows']:
calculated_formula_str = (int(result['rows'][0]['sum_value']) /
int(result['rows'][0]['count_value'])
) * (60 / int(interval_in_secs))
temp_dict = {y: {x: calculated_formula_str}}
resp.append(temp_dict)
rate_dict = {}
for d in resp:
for k, v in d.items():
rate_dict[k] = {**rate_dict[k], **v} if k in rate_dict else v
return web.json_response({"rates": rate_dict})
async def test_add_service_with_config(self, client):
payload = '{"name": "Sine", "type": "south", "plugin": "sinusoid", "enabled": "false",' \
' "config": {"dataPointsPerSec": {"value": "10"}}}'
data = json.loads(payload)
async def async_mock_get_schedule():
schedule = StartUpSchedule()
schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b'
return schedule
@asyncio.coroutine
def q_result(*arg):
table = arg[0]
_payload = arg[1]
if table == 'scheduled_processes':
assert {'return': ['name'],
'where': {'column': 'name', 'condition': '=', 'value': 'south_c'}} == json.loads(_payload)
return {'count': 0, 'rows': []}
if table == 'schedules':
assert {'return': ['schedule_name'],
'where': {'column': 'schedule_name', 'condition': '=',
'value': data['name']}} == json.loads(_payload)
return {'count': 0, 'rows': []}
expected_insert_resp = {'rows_affected': 1, "response": "inserted"}
mock_plugin_info = {
'name': data['name'],
'version': "1.1",
'type': "south",
'interface': "1.0",
'mode': "async",
'config': {
'plugin': {
'description': "Sinusoid Plugin",
'type': 'string',
'default': 'sinusoid'
},
'dataPointsPerSec': {
'description': 'Data points per second',
'type': 'integer',
'default': '1',
'order': '2'
}
}
}
server.Server.scheduler = Scheduler(None, None)
storage_client_mock = MagicMock(StorageClientAsync)
c_mgr = ConfigurationManager(storage_client_mock)
with patch.object(common, 'load_and_fetch_python_plugin_info', side_effect=[mock_plugin_info]):
with patch.object(connect, 'get_storage_async', return_value=storage_client_mock):
with patch.object(c_mgr, 'get_category_all_items', return_value=self.async_mock(None)) as patch_get_cat_info:
with patch.object(storage_client_mock, 'query_tbl_with_payload', side_effect=q_result):
with patch.object(storage_client_mock, 'insert_into_tbl',
return_value=self.async_mock(expected_insert_resp)) as insert_table_patch:
with patch.object(c_mgr, 'create_category', return_value=self.async_mock(None)) as patch_create_cat:
with patch.object(c_mgr, 'create_child_category',
return_value=self.async_mock(None)) as patch_create_child_cat:
with patch.object(c_mgr, 'set_category_item_value_entry',
return_value=self.async_mock(None)) as patch_set_entry:
with patch.object(server.Server.scheduler, 'save_schedule',
return_value=self.async_mock("")) as patch_save_schedule:
with patch.object(server.Server.scheduler, 'get_schedule_by_name',
return_value=async_mock_get_schedule()) as patch_get_schedule:
resp = await client.post('/fledge/service', data=payload)
server.Server.scheduler = None
assert 200 == resp.status
result = await resp.text()
json_response = json.loads(result)
assert {'id': '2129cc95-c841-441a-ad39-6469a87dbc8b',
'name': data['name']} == json_response
patch_get_schedule.assert_called_once_with(data['name'])
patch_save_schedule.called_once_with()
patch_set_entry.assert_called_once_with(data['name'], 'dataPointsPerSec', '10')
patch_create_child_cat.assert_called_once_with('South', ['Sine'])
assert 2 == patch_create_cat.call_count
patch_create_cat.assert_called_with('South', {}, 'South microservices', True)
args, kwargs = insert_table_patch.call_args
assert 'scheduled_processes' == args[0]
p = json.loads(args[1])
assert {'name': 'south_c', 'script': '["services/south_c"]'} == p
patch_get_cat_info.assert_called_once_with(category_name=data['name'])
async def test_add_task_with_config(self, client):
async def async_mock_get_schedule():
schedule = TimedSchedule()
schedule.schedule_id = '2129cc95-c841-441a-ad39-6469a87dbc8b'
return schedule
@asyncio.coroutine
def q_result(*arg):
table = arg[0]
payload = arg[1]
if table == 'scheduled_processes':
assert {
'return': ['name'],
'where': {
'column': 'name',
'condition': '=',
'value': 'north'
}
} == json.loads(payload)
return {'count': 0, 'rows': []}
if table == 'schedules':
assert {
'return': ['schedule_name'],
'where': {
'column': 'schedule_name',
'condition': '=',
'value': 'north bound'
}
} == json.loads(payload)
return {'count': 0, 'rows': []}
if table == 'tasks':
return {'count': 0, 'rows': []}
expected_insert_resp = {'rows_affected': 1, "response": "inserted"}
mock_plugin_info = {
'name': "PI server",
'version': "1.1",
'type': "north",
'interface': "1.0",
'config': {
'plugin': {
'description': "North PI plugin",
'type': 'string',
'default': 'omf'
},
'producerToken': {
'description': 'Producer token for this Fledge stream',
'type': 'string',
'default': 'pi_server_north_0001',
'order': '2'
}
}
}
server.Server.scheduler = Scheduler(None, None)
data = {
"name": "north bound",
"plugin": "omf",
"type": "north",
"schedule_type": 3,
"schedule_day": 0,
"schedule_time": 0,
"schedule_repeat": 30,
"schedule_enabled": True,
"config": {
"producerToken": {
"value": "uid=180905062754237&sig=kx5l+"
}
}
}
storage_client_mock = MagicMock(StorageClientAsync)
c_mgr = ConfigurationManager(storage_client_mock)
with patch.object(common,
'load_and_fetch_python_plugin_info',
side_effect=[mock_plugin_info]):
with patch.object(connect,
'get_storage_async',
return_value=storage_client_mock):
with patch.object(c_mgr,
'get_category_all_items',
return_value=self.async_mock(
None)) as patch_get_cat_info:
with patch.object(storage_client_mock,
'query_tbl_with_payload',
side_effect=q_result):
with patch.object(storage_client_mock, 'insert_into_tbl', return_value=self.async_mock(expected_insert_resp)) \
as insert_table_patch:
with patch.object(c_mgr,
'create_category',
return_value=self.async_mock(
None)) as patch_create_cat:
with patch.object(c_mgr, 'create_child_category', return_value=self.async_mock(None)) \
as patch_create_child_cat:
with patch.object(
c_mgr,
'set_category_item_value_entry',
return_value=self.async_mock(
None)) as patch_set_entry:
with patch.object(
server.Server.scheduler,
'save_schedule',
return_value=self.async_mock(
"")
) as patch_save_schedule:
with patch.object(
server.Server.scheduler,
'get_schedule_by_name',
return_value=
async_mock_get_schedule(
)) as patch_get_schedule:
resp = await client.post(
'/fledge/scheduled/task',
data=json.dumps(data))
server.Server.scheduler = None
assert 200 == resp.status
result = await resp.text()
json_response = json.loads(
result)
assert {
'id':
'2129cc95-c841-441a-ad39-6469a87dbc8b',
'name': 'north bound'
} == json_response
patch_get_schedule.assert_called_once_with(
data['name'])
patch_save_schedule.called_once_with()
patch_set_entry.assert_called_once_with(
data['name'], 'producerToken',
'uid=180905062754237&sig=kx5l+')
patch_create_child_cat.assert_called_once_with(
'North', ['north bound'])
assert 2 == patch_create_cat.call_count
patch_create_cat.assert_called_with(
'North', {}, 'North tasks', True)
args, kwargs = insert_table_patch.call_args
assert 'scheduled_processes' == args[0]
p = json.loads(args[1])
assert p['name'] == 'north'
assert p['script'] == '["tasks/north"]'
patch_get_cat_info.assert_called_once_with(
category_name=data['name'])