async def _get_tracked_plugin_assets_and_readings(storage_client, svc_name):
asset_json = []
payload = PayloadBuilder().SELECT(["asset", "plugin"]).WHERE(['service', '=', svc_name]).\
AND_WHERE(['event', '=', 'Ingest']).payload()
try:
result = await storage_client.query_tbl_with_payload(
'asset_tracker', payload)
asset_records = result['rows']
plugin = ''
if len(result['rows']):
plugin = result['rows'][0]['plugin']
for r in asset_records:
payload = PayloadBuilder().SELECT("value").WHERE(
["key", "=", r["asset"].upper()]).payload()
results = await storage_client.query_tbl_with_payload(
"statistics", payload)
if int(results['count']):
asset_result = results['rows'][0]
asset_json.append({
"count": asset_result['value'],
"asset": r["asset"]
})
except:
raise
else:
return plugin, asset_json
def prepare_limit_skip_payload(request, _dict):
""" limit skip clause validation
Args:
request: request query params
_dict: main payload dict
Returns:
chain payload dict
"""
limit = __DEFAULT_LIMIT
if 'limit' in request.query and request.query['limit'] != '':
try:
limit = int(request.query['limit'])
if limit < 0:
raise ValueError
except ValueError:
raise web.HTTPBadRequest(reason="Limit must be a positive integer")
offset = __DEFAULT_OFFSET
if 'skip' in request.query and request.query['skip'] != '':
try:
offset = int(request.query['skip'])
if offset < 0:
raise ValueError
except ValueError:
raise web.HTTPBadRequest(
reason="Skip/Offset must be a positive integer")
payload = PayloadBuilder(_dict).LIMIT(limit)
if offset:
payload = PayloadBuilder(_dict).SKIP(offset)
return payload.chain_payload()
def test_expr_payload(self):
res = PayloadBuilder().WHERE(["key", "=", "READINGS"]).EXPR(["value", "+", 10]).payload()
assert _payload("data/payload_expr1.json") == json.loads(res)
exprs = (["value1", "+", 10], ["value2", "-", 5]) # a tuple
res = PayloadBuilder().WHERE(["key", "=", "READINGS"]).EXPR(exprs).payload()
assert 2 == len(json.loads(res))
assert _payload("data/payload_expr2.json") == json.loads(res)
async def asset_datapoints_with_bucket_size(
request: web.Request) -> web.Response:
""" Retrieve datapoints for an asset.
If bucket_size is not given then the bucket size is 1
If start is not given then the start point is now - 60 seconds.
If length is not given then length is 60 seconds. And length is calculated with length / bucket_size
For multiple assets use comma separated values in request and this will allow data from one or more asset to be returned.
:Example:
curl -sX GET http://localhost:8081/fledge/asset/{asset_code}/bucket/{bucket_size}
curl -sX GET http://localhost:8081/fledge/asset/{asset_code_1},{asset_code_2}/bucket/{bucket_size}
"""
try:
asset_code = request.match_info.get('asset_code', '')
bucket_size = request.match_info.get('bucket_size', 1)
length = 60
ts = datetime.datetime.now().timestamp()
start = ts - 60
asset_code_list = asset_code.split(',')
_readings = connect.get_readings_async()
for code in asset_code_list:
verify_asset_payload = PayloadBuilder().WHERE(["asset_code", "in", [code]]).LIMIT(1).\
ORDER_BY(["user_ts", "desc"]).payload()
res = await _readings.query(verify_asset_payload)
if not res['rows']:
raise KeyError("{} asset code not found".format(code))
if 'start' in request.query and request.query['start'] != '':
start = float(request.query['start'])
if start < 0:
raise ValueError('start must be a positive integer')
_aggregate = PayloadBuilder().AGGREGATE(["all"]).chain_payload()
_and_where = PayloadBuilder(_aggregate).WHERE(
["asset_code", "in",
asset_code_list]).AND_WHERE(["user_ts", ">=",
str(start)]).chain_payload()
_bucket = PayloadBuilder(_and_where).TIMEBUCKET(
'user_ts', bucket_size, 'YYYY-MM-DD HH24:MI:SS',
'timestamp').chain_payload()
if 'length' in request.query and request.query['length'] != '':
length = int(request.query['length'])
if length < 0:
raise ValueError('length must be a positive integer')
payload = PayloadBuilder(_bucket).LIMIT(int(
length / int(bucket_size))).payload()
# Sort & timebucket modifiers can not be used in same payload
# payload = PayloadBuilder(limit).ORDER_BY(["user_ts", "desc"]).payload()
results = await _readings.query(payload)
response = results['rows']
except (KeyError, IndexError) as e:
raise web.HTTPNotFound(reason=e)
except (TypeError, ValueError) as e:
raise web.HTTPBadRequest(reason=e)
except Exception as e:
raise web.HTTPInternalServerError(reason=str(e))
else:
return web.json_response(response)
async def asset_summary(request):
""" Browse all the assets for which we have recorded readings and
return a summary for a particular sensor. The values that are
returned are the min, max and average values of the sensor.
The readings summarised can also be time limited by use of the query
parameter seconds=sss. This defines a number of seconds that the reading
must have been processed in. Older readings than this will not be summarised.
The readings summarised can also be time limited by use of the query
parameter minutes=mmm. This defines a number of minutes that the reading
must have been processed in. Older readings than this will not be summarised.
The readings summarised can also be time limited by use of the query
parameter hours=hh. This defines a number of hours that the reading
must have been processed in. Older readings than this will not be summarised.
Only one of hour, minutes or seconds should be supplied
Returns:
json result on basis of SELECT MIN(reading->>'reading'), MAX(reading->>'reading'), AVG((reading->>'reading')::float) FROM readings WHERE asset_code = 'asset_code';
:Example:
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/temperature/summary
"""
asset_code = request.match_info.get('asset_code', '')
reading = request.match_info.get('reading', '')
try:
payload = PayloadBuilder().SELECT("reading").WHERE(
["asset_code", "=",
asset_code]).LIMIT(1).ORDER_BY(["user_ts", "desc"]).payload()
_readings = connect.get_readings_async()
results = await _readings.query(payload)
if not results['rows']:
raise web.HTTPNotFound(
reason="{} asset_code not found".format(asset_code))
# TODO: FOGL-1768 when support available from storage layer then avoid multiple calls
reading_keys = list(results['rows'][-1]['reading'].keys())
if reading not in reading_keys:
raise web.HTTPNotFound(
reason="{} reading key is not found".format(reading))
_aggregate = PayloadBuilder().AGGREGATE(["min", ["reading", reading]], ["max", ["reading", reading]],
["avg", ["reading", reading]]) \
.ALIAS('aggregate', ('reading', 'min', 'min'), ('reading', 'max', 'max'),
('reading', 'avg', 'average')).chain_payload()
_where = PayloadBuilder(_aggregate).WHERE(
["asset_code", "=", asset_code]).chain_payload()
_and_where = where_clause(request, _where)
payload = PayloadBuilder(_and_where).payload()
results = await _readings.query(payload)
# for aggregates, so there can only ever be one row
response = results['rows'][0]
except KeyError:
raise web.HTTPBadRequest(reason=results['message'])
else:
return web.json_response({reading: response})
async def asset_all_readings_summary(request):
""" Browse all the assets for which we have recorded readings and
return a summary for all sensors values for an asset code. The values that are
returned are the min, max and average values of the sensor.
Only one of hour, minutes or seconds should be supplied, if more than one time unit
then the smallest unit will be picked
The number of records return is default to a small number (20), this may be changed by supplying
the query parameter ?limit=xx&skip=xx and it will not respect when datetime units is supplied
:Example:
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/summary
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/summary?seconds=60
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/summary?limit=10
"""
try:
# Get readings from asset_code
asset_code = request.match_info.get('asset_code', '')
# TODO: Use only the latest asset read to determine the data points to use. This
# avoids reading every single reading into memory and creating a very big result set See FOGL-2635
payload = PayloadBuilder().SELECT("reading").WHERE(
["asset_code", "=",
asset_code]).LIMIT(1).ORDER_BY(["user_ts", "desc"]).payload()
_readings = connect.get_readings_async()
results = await _readings.query(payload)
if not results['rows']:
raise web.HTTPNotFound(
reason="{} asset_code not found".format(asset_code))
# TODO: FOGL-1768 when support available from storage layer then avoid multiple calls
# Find keys in readings
reading_keys = list(results['rows'][-1]['reading'].keys())
response = []
_where = PayloadBuilder().WHERE(["asset_code", "=",
asset_code]).chain_payload()
if 'seconds' in request.query or 'minutes' in request.query or 'hours' in request.query:
_and_where = where_clause(request, _where)
else:
# Add limit, offset clause
_and_where = prepare_limit_skip_payload(request, _where)
for reading in reading_keys:
_aggregate = PayloadBuilder(_and_where).AGGREGATE(["min", ["reading", reading]],
["max", ["reading", reading]],
["avg", ["reading", reading]]) \
.ALIAS('aggregate', ('reading', 'min', 'min'),
('reading', 'max', 'max'),
('reading', 'avg', 'average')).chain_payload()
payload = PayloadBuilder(_aggregate).payload()
results = await _readings.query(payload)
response.append({reading: results['rows'][0]})
except (KeyError, IndexError) as ex:
raise web.HTTPNotFound(reason=ex)
except (TypeError, ValueError) as ex:
raise web.HTTPBadRequest(reason=ex)
else:
return web.json_response(response)
async def asset_reading(request):
""" Browse a particular sensor value of a particular asset for which we have recorded readings and
return the timestamp and reading value for that sensor. The number of rows returned
is limited to a small number, this number may be altered by use of
the query parameter limit=xxx&skip=xxx and it will not respect when datetime units is supplied
The readings returned can also be time limited by use of the query
parameter seconds=sss. This defines a number of seconds that the reading
must have been processed in. Older readings than this will not be returned.
The readings returned can also be time limited by use of the query
parameter minutes=mmm. This defines a number of minutes that the reading
must have been processed in. Older readings than this will not be returned.
The readings returned can also be time limited by use of the query
parameter hours=hh. This defines a number of hours that the reading
must have been processed in. Older readings than this will not be returned.
Only one of hour, minutes or seconds should be supplied
Returns:
json result on basis of SELECT user_ts as "timestamp", reading->>'reading' FROM readings WHERE asset_code = 'asset_code' ORDER BY user_ts DESC LIMIT 20 OFFSET 0;
:Example:
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/temperature
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/temperature?limit=1
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/temperature?skip=10
curl -sX GET "http://localhost:8081/fledge/asset/fogbench_humidity/temperature?limit=1&skip=10"
curl -sX GET http://localhost:8081/fledge/asset/fogbench_humidity/temperature?minutes=60
"""
asset_code = request.match_info.get('asset_code', '')
reading = request.match_info.get('reading', '')
_select = PayloadBuilder().SELECT(("user_ts", ["reading", reading])) \
.ALIAS("return", ("user_ts", "timestamp"), ("reading", reading)).chain_payload()
_where = PayloadBuilder(_select).WHERE(["asset_code", "=",
asset_code]).chain_payload()
if 'seconds' in request.query or 'minutes' in request.query or 'hours' in request.query:
_and_where = where_clause(request, _where)
else:
# Add the order by and limit, offset clause
_and_where = prepare_limit_skip_payload(request, _where)
payload = PayloadBuilder(_and_where).ORDER_BY(["user_ts",
"desc"]).payload()
results = {}
try:
_readings = connect.get_readings_async()
results = await _readings.query(payload)
response = results['rows']
except KeyError:
raise web.HTTPBadRequest(reason=results['message'])
else:
return web.json_response(response)
async def _insert_pwd_history_with_oldest_pwd_deletion_if_count_exceeds(
cls, storage_client, user_id, hashed_pwd, pwd_history_list):
# delete oldest password for user, as storage result in sorted order so its safe to delete its last index from pwd_history_list
if len(pwd_history_list) >= USED_PASSWORD_HISTORY_COUNT:
payload = PayloadBuilder().WHERE(
['user_id', '=',
user_id]).AND_WHERE(['pwd', '=',
pwd_history_list[-1]]).payload()
await storage_client.delete_from_tbl("user_pwd_history",
payload)
# insert into password history table
payload = PayloadBuilder().INSERT(user_id=user_id,
pwd=hashed_pwd).payload()
await storage_client.insert_into_tbl("user_pwd_history", payload)
def test_timebucket(self, test_input, expected):
timestamp = test_input[0]
fmt = None
alias = None
if len(test_input) == 1:
res = PayloadBuilder().TIMEBUCKET(timestamp).payload()
else:
size = test_input[1]
if len(test_input) == 3:
fmt = test_input[2]
if len(test_input) == 4:
fmt = test_input[2]
alias = test_input[3]
res = PayloadBuilder().TIMEBUCKET(timestamp, size, fmt, alias).payload()
assert expected == json.loads(res)
async def delete(cls, user_id):
"""
Args:
user_id: user id to delete
Returns:
json response
"""
# either keep 1 admin user or just reserve id:1 for superuser
if int(user_id) == 1:
raise ValueError("Super admin user can not be deleted")
storage_client = connect.get_storage_async()
try:
# first delete the active login references
await cls.delete_user_tokens(user_id)
payload = PayloadBuilder().SET(enabled="f").WHERE(
['id', '=', user_id]).AND_WHERE(['enabled', '=',
't']).payload()
result = await storage_client.update_tbl("users", payload)
except StorageServerError as ex:
if ex.error["retryable"]:
pass # retry INSERT
raise ValueError(ERROR_MSG)
return result
async def update_bulk(self, stat_list):
""" Bulk update statistics table keys and their values
Args:
stat_list: dict containing statistics keys and increment values
Returns:
None
"""
if not isinstance(stat_list, dict):
raise TypeError('stat_list must be a dict')
try:
payload = {"updates": []}
for k, v in stat_list.items():
payload_item = PayloadBuilder() \
.WHERE(["key", "=", k]) \
.EXPR(["value", "+", v]) \
.payload()
payload['updates'].append(json.loads(payload_item))
await self._storage.update_tbl(
"statistics", json.dumps(payload, sort_keys=False))
except Exception as ex:
_logger.exception('Unable to bulk update statistics %s', str(ex))
raise
async def update(self, key, value_increment):
""" UPDATE the value column only of a statistics row based on key
Args:
key: statistics key value (required)
value_increment: amount to increment the value by
Returns:
None
"""
if not isinstance(key, str):
raise TypeError('key must be a string')
if not isinstance(value_increment, int):
raise ValueError('value must be an integer')
try:
payload = PayloadBuilder()\
.WHERE(["key", "=", key])\
.EXPR(["value", "+", value_increment])\
.payload()
await self._storage.update_tbl("statistics", payload)
except Exception as ex:
_logger.exception(
'Unable to update statistics value based on statistics_key %s and value_increment %d, error %s',
key, value_increment, str(ex))
raise
async def run(self):
""" SELECT against the statistics table, to get a snapshot of the data at that moment.
Based on the snapshot:
1. INSERT the delta between `value` and `previous_value` into statistics_history
2. UPDATE the previous_value in statistics table to be equal to statistics.value at snapshot
"""
current_time = common_utils.local_timestamp()
results = await self._storage_async.query_tbl("statistics")
# Bulk updates payload
payload = {"updates": []}
# Bulk inserts payload
insert_payload = {"inserts": []}
for r in results['rows']:
key = r['key']
value = int(r["value"])
previous_value = int(r["previous_value"])
delta = value - previous_value
payload_item = PayloadBuilder().SET(previous_value=value).WHERE(
["key", "=", key]).payload()
# Add element to bulk updates
payload['updates'].append(json.loads(payload_item))
# Add element to bulk inserts
insert_payload['inserts'].append({
'key': key,
'value': delta,
'history_ts': current_time
})
# Bulk inserts
await self._storage_async.insert_into_tbl("statistics_history",
json.dumps(insert_payload))
# Bulk updates
await self._bulk_update_previous_value(payload)
async def get_scheduled_process(request):
"""
Returns:
a list of all the defined scheduled_processes from scheduled_processes table
:Example:
curl -X GET http://localhost:8081/fledge/schedule/process/purge
curl -X GET http://localhost:8081/fledge/schedule/process/purge%2Cbackup%2Crestore
curl -X GET http://localhost:8081/fledge/schedule/process/purge%2Cbackup%2Cstats%20collector
"""
scheduled_process_names = request.match_info.get('scheduled_process_name',
None)
scheduled_process_name = scheduled_process_names.split(',')
payload = PayloadBuilder().SELECT("name").WHERE(
["name", "in", scheduled_process_name]).payload()
_storage = connect.get_storage_async()
scheduled_process = await _storage.query_tbl_with_payload(
'scheduled_processes', payload)
if len(scheduled_process['rows']) == 0:
raise web.HTTPNotFound(reason='No such Scheduled Process: {}.'.format(
scheduled_process_name))
if len(scheduled_process['rows']) == 1:
retval = scheduled_process['rows'][0].get("name")
else:
retval = scheduled_process['rows']
return web.json_response(retval)
async def certificate_login(cls, username, host):
"""
Args:
username: username
host: IP address
Returns:
uid: User id
token: jwt token
is_admin: boolean flag
"""
storage_client = connect.get_storage_async()
# get user info on the basis of username
payload = PayloadBuilder().SELECT("id", "role_id").WHERE(['uname', '=', username])\
.AND_WHERE(['enabled', '=', 't']).payload()
result = await storage_client.query_tbl_with_payload(
'users', payload)
if len(result['rows']) == 0:
raise User.DoesNotExist('User does not exist')
found_user = result['rows'][0]
uid, jwt_token, is_admin = await cls._get_new_token(
storage_client, found_user, host)
return uid, jwt_token, is_admin
async def all(cls):
storage_client = connect.get_storage_async()
payload = PayloadBuilder().SELECT("id", "uname", "role_id").WHERE(
['enabled', '=', 't']).payload()
result = await storage_client.query_tbl_with_payload(
'users', payload)
return result['rows']
async def asset_counts(request):
""" Browse all the assets for which we have recorded readings and
return a readings count.
Returns:
json result on basis of SELECT asset_code, count(*) FROM readings GROUP BY asset_code;
:Example:
curl -sX GET http://localhost:8081/fledge/asset
"""
payload = PayloadBuilder().AGGREGATE(["count", "*"]).ALIAS("aggregate", ("*", "count", "count")) \
.GROUP_BY("asset_code").payload()
results = {}
try:
_readings = connect.get_readings_async()
results = await _readings.query(payload)
response = results['rows']
asset_json = [{
"count": r['count'],
"assetCode": r['asset_code']
} for r in response]
except KeyError:
raise web.HTTPBadRequest(reason=results['message'])
else:
return web.json_response(asset_json)
async def _get_new_token(cls, storage_client, found_user, host):
# fetch user info
exp = datetime.now() + timedelta(seconds=JWT_EXP_DELTA_SECONDS)
uid = found_user['id']
p = {'uid': uid, 'exp': exp}
jwt_token = jwt.encode(p, JWT_SECRET,
JWT_ALGORITHM).decode("utf-8")
payload = PayloadBuilder().INSERT(user_id=p['uid'],
token=jwt_token,
token_expiration=str(exp),
ip=host).payload()
# Insert token, uid, expiration into user_login table
try:
await storage_client.insert_into_tbl("user_logins", payload)
except StorageServerError as ex:
if ex.error["retryable"]:
pass # retry INSERT
raise ValueError(ERROR_MSG)
# TODO remove hard code role id to return is_admin info
if int(found_user['role_id']) == 1:
return uid, jwt_token, True
return uid, jwt_token, False
async def _log(self, level, code, log):
try:
if log is None:
payload = PayloadBuilder().INSERT(code=code,
level=level).payload()
else:
payload = PayloadBuilder().INSERT(code=code,
level=level,
log=log).payload()
await self._storage.insert_into_tbl("log", payload)
except (StorageServerError, Exception) as ex:
_logger.exception("Failed to log audit trail entry '%s': %s", code,
str(ex))
raise ex
async def add_update(self, sensor_stat_dict):
"""UPDATE the value column of a statistics based on key, if key is not present, ADD the new key
Args:
sensor_stat_dict: Dictionary containing the key value of Asset name and value increment
Returns:
None
"""
for key, value_increment in sensor_stat_dict.items():
# Try updating the statistics value for given key
try:
payload = PayloadBuilder() \
.WHERE(["key", "=", key]) \
.EXPR(["value", "+", value_increment]) \
.payload()
result = await self._storage.update_tbl("statistics", payload)
if result["response"] != "updated":
raise KeyError
# If key was not present, add the key and with value = value_increment
except KeyError:
_logger.exception('Statistics key %s has not been registered',
key)
raise
except Exception as ex:
_logger.exception(
'Unable to update statistics value based on statistics_key %s and value_increment %s, error %s',
key, value_increment, str(ex))
raise
async def _add_child_filters(storage: StorageClientAsync, cf_mgr: ConfigurationManager, user_name: str,
filter_list: List[str], old_list: List[str] = []) -> None:
# Create children categories. Since create_category() does not expect "value" key to be
# present in the payload, we need to remove all "value" keys BUT need to add back these
# "value" keys to the new configuration.
for filter_name in filter_list:
filter_config = await cf_mgr.get_category_all_items(category_name="{}_{}".format(user_name, filter_name))
# If "username_filter" category does not exist
if filter_config is None:
filter_config = await cf_mgr.get_category_all_items(category_name=filter_name)
filter_desc = "Configuration of {} filter for user {}".format(filter_name, user_name)
new_filter_config, deleted_values = _delete_keys_from_dict(filter_config, ['value'], deleted_values={}, parent=None)
await cf_mgr.create_category(category_name="{}_{}".format(user_name, filter_name),
category_description=filter_desc,
category_value=new_filter_config,
keep_original_items=True)
if deleted_values != {}:
await cf_mgr.update_configuration_item_bulk("{}_{}".format(user_name, filter_name), deleted_values)
# Remove cat from cache
if filter_name in cf_mgr._cacheManager.cache:
cf_mgr._cacheManager.remove(filter_name)
# Create children categories in category_children table
children = ["{}_{}".format(user_name, _filter) for _filter in filter_list]
await cf_mgr.create_child_category(category_name=user_name, children=children)
# Add entries to filter_users table
new_added = _diff(old_list, filter_list)
for filter_name in new_added:
payload = PayloadBuilder().INSERT(name=filter_name, user=user_name).payload()
await storage.insert_into_tbl("filter_users", payload)
async def purge_audit_trail_log(self, config):
"""" Purge log table based on the Age which is defined under in config item
"""
ts = datetime.now() - timedelta(
days=int(config['retainAuditLog']['value']))
payload = PayloadBuilder().WHERE(['ts', '<=', str(ts)]).payload()
await self._storage_async.delete_from_tbl("log", payload)
async def get_role_id_by_name(cls, name):
storage_client = connect.get_storage_async()
payload = PayloadBuilder().SELECT("id").WHERE(['name', '=',
name]).payload()
result = await storage_client.query_tbl_with_payload(
'roles', payload)
return result["rows"]
async def get_filter(request: web.Request) -> web.Response:
""" GET filter detail
:Example:
curl -X GET http://localhost:8081/fledge/filter/<filter_name>
"""
filter_name = request.match_info.get('filter_name', None)
try:
storage = connect.get_storage_async()
cf_mgr = ConfigurationManager(storage)
filter_detail = {}
# Fetch the filter items: get category items
category_info = await cf_mgr.get_category_all_items(filter_name)
if category_info is None:
raise ValueError(
"No such '{}' category found.".format(filter_name))
filter_detail.update({"config": category_info})
# Fetch filter detail
payload = PayloadBuilder().WHERE(['name', '=', filter_name]).payload()
result = await storage.query_tbl_with_payload("filters", payload)
if len(result["rows"]) == 0:
raise ValueError("No such filter '{}' found.".format(filter_name))
row = result["rows"][0]
filter_detail.update({"name": row["name"], "plugin": row["plugin"]})
# Fetch users which are using this filter
payload = PayloadBuilder().WHERE(['name', '=', filter_name]).payload()
result = await storage.query_tbl_with_payload("filter_users", payload)
users = []
for row in result["rows"]:
users.append(row["user"])
filter_detail.update({"users": users})
except StorageServerError as ex:
_LOGGER.exception("Get filter: %s, caught exception: %s", filter_name,
str(ex.error))
raise web.HTTPInternalServerError(reason=str(ex.error))
except ValueError as ex:
raise web.HTTPNotFound(reason=ex)
except TypeError as ex:
raise web.HTTPBadRequest(reason=ex)
except Exception as ex:
raise web.HTTPInternalServerError(reason=ex)
else:
return web.json_response({'filter': filter_detail})
async def _delete_configuration_category(storage: StorageClientAsync, key: str) -> None:
payload = PayloadBuilder().WHERE(['key', '=', key]).payload()
await storage.delete_from_tbl('configuration', payload)
# Removed category from configuration cache and other related stuff e.g. script files
config_mgr = ConfigurationManager(storage)
config_mgr.delete_category_related_things(key)
config_mgr._cacheManager.remove(key)
async def _get_plugin_and_sch_name_from_asset_tracker(_type: str) -> list:
event_name = "Ingest" if _type == "south" else "Egress"
storage_client = connect.get_storage_async()
payload = PayloadBuilder().SELECT("plugin", "service").WHERE(
['event', '=', event_name]).payload()
result = await storage_client.query_tbl_with_payload(
'asset_tracker', payload)
return result['rows']
def test_chain_payload(self):
res_chain = PayloadBuilder() \
.SELECT("id", "name") \
.WHERE(["id", "=", 1]) \
.AND_WHERE(["name", "=", "test"]) \
.OR_WHERE(["name", "=", "test2"]) \
.chain_payload()
res = PayloadBuilder(res_chain) \
.LIMIT(5) \
.OFFSET(1) \
.GROUP_BY("name", "id") \
.ORDER_BY(["id", "desc"]) \
.AGGREGATE(["count", "name"]) \
.payload()
assert _payload("data/payload_complex_select1.json") == json.loads(res)
async def update_service(request: web.Request) -> web.Response:
""" update service
:Example:
curl -sX PUT http://localhost:8081/fledge/service/notification/notification/update
"""
_type = request.match_info.get('type', None)
name = request.match_info.get('name', None)
try:
_type = _type.lower()
if _type != 'notification':
raise ValueError("Invalid service type. Must be 'notification'")
# Check requested service name is installed or not
installed_services = get_service_installed()
if name not in installed_services:
raise KeyError(
"{} service is not installed yet. Hence update is not possible."
.format(name))
storage_client = connect.get_storage_async()
# TODO: process_name ends with "_c" suffix
payload = PayloadBuilder().SELECT(
"id", "enabled",
"schedule_name").WHERE(['process_name', '=',
'{}_c'.format(_type)]).payload()
result = await storage_client.query_tbl_with_payload(
'schedules', payload)
sch_info = result['rows']
sch_list = []
if sch_info and sch_info[0]['enabled'] == 't':
status, reason = await server.Server.scheduler.disable_schedule(
uuid.UUID(sch_info[0]['id']))
if status:
_logger.warning(
"Schedule is disabled for {}, as {} service of type {} is being updated..."
.format(sch_info[0]['schedule_name'], name, _type))
# TODO: SCHCH Audit log entry
sch_list.append(sch_info[0]['id'])
# service update is running as a background task
loop = request.loop
request._type = _type
request._name = name
request._sch_list = sch_list
loop.call_later(1, do_update, request)
except KeyError as ex:
raise web.HTTPNotFound(reason=str(ex))
except ValueError as ex:
raise web.HTTPBadRequest(reason=str(ex))
except Exception as ex:
raise web.HTTPInternalServerError(reason=str(ex))
return web.json_response({
"message":
"{} service update in process. Wait for few minutes to complete.".
format(name)
})
async def purge_stats_history(self, config):
"""" Purge statistics history table based on the Age which is defined in retainStatsHistory config item
"""
ts = datetime.now() - timedelta(
days=int(config['retainStatsHistory']['value']))
payload = PayloadBuilder().WHERE(['history_ts', '<=',
str(ts)]).payload()
await self._storage_async.delete_from_tbl("statistics_history",
payload)
async def delete_filter(request: web.Request) -> web.Response:
""" DELETE filter
:Example:
curl -X DELETE http://localhost:8081/fledge/filter/<filter_name>
"""
filter_name = request.match_info.get('filter_name', None)
try:
storage = connect.get_storage_async()
# Check if it is a valid plugin
payload = PayloadBuilder().WHERE(['name', '=', filter_name]).payload()
result = await storage.query_tbl_with_payload("filters", payload)
if len(result["rows"]) == 0:
raise ValueError("No such filter '{}' found".format(filter_name))
# Check if filter exists in any pipeline
payload = PayloadBuilder().WHERE(['name', '=', filter_name]).payload()
result = await storage.query_tbl_with_payload("filter_users", payload)
if len(result["rows"]) != 0:
raise TypeError(
"Filter '{}' found in pipelines".format(filter_name))
# Delete filter from filters table
payload = PayloadBuilder().WHERE(['name', '=', filter_name]).payload()
await storage.delete_from_tbl("filters", payload)
# Delete configuration for filter
await _delete_configuration_category(storage, filter_name)
except StorageServerError as ex:
_LOGGER.exception("Delete filter: %s, caught exception: %s",
filter_name, str(ex.error))
raise web.HTTPInternalServerError(reason=str(ex.error))
except ValueError as ex:
raise web.HTTPNotFound(reason=ex)
except TypeError as ex:
raise web.HTTPBadRequest(reason=ex)
except Exception as ex:
raise web.HTTPInternalServerError(reason=ex)
else:
return web.json_response(
{'result': "Filter {} deleted successfully".format(filter_name)})
