def stage_attributes(self, attribute: str, unit_value: str,
bespoke_unit_tag: int, bespoke_sub_theme: int,
description: str) -> db.Model:
"""
Stage Attributes
:param attribute: Attribute
:param unit_value: Unit Value
:param bespoke_unit_tag: Unit Id
:param bespoke_sub_theme: SubTheme Id
:param description: Attributes Description
:return: Attribute instance
"""
_a = Attributes._get_by_name_unit_unitvalue(attribute,
bespoke_unit_tag,
unit_value)
if _a:
logger.info(
'{} attribute with Unit ID: {} and Unit Value: {} already exists'
.format(attribute, str(bespoke_unit_tag), unit_value))
return _a
a = Attributes(id=str(uuid.uuid4()),
name=attribute,
table_name=(attribute + '_' +
str(uuid.uuid4()).replace('-', '_')),
sub_theme=bespoke_sub_theme,
unit=bespoke_unit_tag,
unit_value=str(unit_value),
description=description)
a = a.save()
return a
def test_return_attribute_of_subtheme(self):
"""
Adding a theme and subtheme to the database and then testing to see if it's data is
retrieved correctly
"""
theme = Theme("Test_Theme")
theme.save()
theme.commit()
sub_theme = SubTheme(theme.id, "Test_Sub_Theme")
sub_theme.save()
sub_theme.commit()
attributes = Attributes("1234567890-123456789-123456789",
"_test_attribute_", "_table_name_",
sub_theme.id, 1)
attributes.save()
attributes.commit()
response = self.testing_client.get('/data',
data=dict(subtheme=theme.id))
self.assertEqual(theme.json(), response.get_json())
attributes.delete()
attributes.commit()
sub_theme.delete()
sub_theme.commit()
theme.delete()
theme.commit()
def attribute_to_commit(self, attribute_data):
# Stage Attribute commit
for attr in attribute_data:
a = Attributes(name=attr.attribute, sub_theme=attr.sub_theme, table_name=str(uuid.uuid4()), unit_value=attr.unit_value, description=attr.description, unit=attr.unit_type)
a.save()
attr.id = a.id
self._print_in_middle('Saved Attributes')
def get(self):
args = self.parser.parse_args()
sensor, sensor_name, sensor_attribute = None, None, None
if 'sensor' in args and args['sensor'] is not None:
sensor = args['sensor']
if sensor != '':
if sensor == 'all':
sensors = Sensor.get_all()
return [a.json() for a in sensors], 200
else:
return (Sensor.get_by_id(sensor)).json(), 200
if 'sensorname' in args and args['sensorname'] is not None:
sensor_name = args['sensorname']
if sensor_name != '':
_sensors = sensor_name.split(',')
_by_name = Sensor.get_by_name_in(_sensors)
return [a.json() for a in _by_name], 200
if 'sensorattribute' in args and args['sensorattribute'] is not None:
sensor_attribute = args['sensorattribute']
if sensor_attribute != '':
_sen_attrs_ids = sensor_attribute.split(',')
_sen_attrs = SensorAttribute.get_by_id_in(_sen_attrs_ids)
attrs_ids = [_id.a_id for _id in _sen_attrs]
_attributes = Attributes.get_by_id_in(attrs_ids)
return [a.json() for a in _attributes], 200
return {
"error": "error occured while processing request"
}, 400
def test_update_on_new_import(self):
"""
Test whether a AttributeRange entry is updated when new values are
imported
"""
o3 = Attributes.get_by_name("O3")[0]
o3_attr_range = AttributeRange.get_by_attr_id(o3.id)
before_import_latest_update = o3_attr_range.latest_update
self.sess.execute(
"INSERT INTO {} VALUES('A','99999999','{}','{}')".format(
o3.table_name.lower(), datetime.now(), datetime.now()))
self.sess.commit()
response = self.testing_client.post('/importer_retry', data=dict(
api_id=1), headers=self.access_token_header)
self.assertEqual(response.status_code, 200)
time.sleep(1)
new_o3_attr_range = AttributeRange.get_by_attr_id(o3.id)
self.assertEqual(new_o3_attr_range.maximum, 99999999)
after_import_latest_update = new_o3_attr_range.latest_update
self.assertGreater(after_import_latest_update,
before_import_latest_update)
self.sess.execute(
"delete from {} where s_id='A'".format(o3.table_name.lower()))
self.sess.commit()
def run():
""" Schedule main_task to execute once a day """
apis = Scheduler.get_apis()
for api in apis:
if not ImporterStatuses.find_by_api_id(api.id):
class_name = api.api_class.split('.')[2]
new_entry = ImporterStatuses(api.id, class_name, 'pending', '',
'', False, datetime.now())
new_entry.save()
new_entry.commit()
all_attribute = Attributes.get_all()
for attribute in all_attribute:
if not AttributeRange.get_by_attr_id(attribute.id):
attr_min = Attributes.attribute_min(attribute.table_name)
attr_max = Attributes.attribute_max(attribute.table_name)
try:
new_range = AttributeRange(attribute.id, attr_min.s_id,
attr_min.value,
attr_min.timestamp,
attr_max.s_id, attr_max.value,
attr_max.timestamp,
datetime.now())
except AttributeError:
new_range = AttributeRange(attribute.id, None, None,
None, None, None, None,
datetime.now())
new_range.save()
new_range.commit()
sched.add_job(Scheduler.main_task,
'interval',
start_date=datetime.now() + timedelta(seconds=5),
days=1,
name='Primary_Scheduler',
replace_existing=True,
id='Primary_Scheduler',
jobstore='sqlalchemy')
try:
# This is here to simulate application activity (which keeps
# the main thread alive).
while True:
time.sleep(2)
except (KeyboardInterrupt, SystemExit):
sched.shutdown()
def setUp(self) -> None:
"""
Setup FlaskClient for tests, create an admin user and create the authorization header for requests to
the FlaskClient
"""
self.clean_up = []
self.client, self.app_context = self.create_test_client()
self.user = self.create_admin_user()
self.theme = self.create_dummy_theme()
self.subtheme = self.create_dummy_subtheme()
self.unit = self.get_dummy_unit()
self.attribute = Attributes("A_TEST_ATTRIBUTE_ID_",
"_TEST_ATTR_NAME_",
"_A_TABLE_NAME",
self.subtheme.id,
self.unit.id,
description="_A_CUSTOM_DESCRIPTION_")
self.attribute.save()
self.attribute.commit()
self.clean_up.append(self.attribute)
def tearDown(self) -> None:
""" Clean up all dependencies after tests"""
for alias in self.aliases:
try:
if AttrAlias.get_by_user_id(self.user.id):
alias.delete()
alias.commit()
except Exception:
pass
for attr in self.attributes:
try:
if Attributes.get_by_id(attr.id):
attr.delete()
attr.commit()
except Exception:
pass
for sub_theme in self.sub_themes:
try:
if SubTheme.get_by_id(sub_theme.id):
sub_theme.delete()
sub_theme.commit()
except Exception:
pass
for unit in self.units:
try:
if Unit.get_by_id(unit.id):
unit.delete()
unit.commit()
except Exception:
pass
try:
if Theme.get_by_id(self.theme.id):
self.theme.delete()
self.theme.commit()
except Exception:
pass
self.client.post('/logout', headers=self.auth_header)
if self.user:
if Users.find_by_id(self.user.id):
try:
self.user.delete()
self.user.commit()
except Exception:
pass
self.app_context.pop()
def setUp(self):
"""
Setup a FlaskClient, AppContext, Admin user, Authorization header for requests to
the Flask Client and a dummy Theme
"""
self.client, self.app_context = self.create_test_client()
self.user = self.create_admin_user()
self.auth_header = self.get_auth_header()
self.unit = Unit("_test_unit_type_", "A test unit")
self.unit.save()
self.unit.commit()
self.attribute = Attributes.get_by_name("_test_attribute_")
self.theme = self.create_dummy_theme()
self.subtheme = self.create_dummy_subtheme()
if not self.attribute:
self.attribute = Attributes("1234567890-123456789-123456789", "_test_attribute_", "_table_name_",
self.subtheme.id, self.unit.id)
self.attribute.save()
self.attribute.commit()
self.clean_up = [self.user, self.attribute, self.theme, self.subtheme, self.theme, self.unit]
def stage_attributes(self, attribute: str, unit_value: str,
bespoke_unit_tag: int, bespoke_sub_theme: int,
description: str):
_a = Attributes._get_by_name_unit_unitvalue(attribute,
bespoke_unit_tag,
unit_value)
if _a:
print(attribute, 'attribute with Unit ID:', str(bespoke_unit_tag),
'and Unit Value:', unit_value, 'already exists')
return _a
a = Attributes(id=str(uuid.uuid4()),
name=attribute,
table_name=(attribute + '_' +
str(uuid.uuid4()).replace('-', '_')),
sub_theme=bespoke_sub_theme,
unit=bespoke_unit_tag,
unit_value=str(unit_value),
description=description)
a = a.save()
return a
def create_attributes(self, count: int) -> None:
"""
Create Attributes for SubThemes
:param count: Number of Attributes per SubTheme
"""
number_attributes = len(self.sub_themes) * count
try:
index = 0
sub_theme = self.sub_themes[index]
for num in range(0, number_attributes):
unit = self.create_unit(num)
attr = Attributes(
"attribute_id_{}".format(num),
"attribute_name_{}".format(num),
"b3_heat_value_bffc4e56_20e2_41a5_84d8_de725a3f875b",
sub_theme.id, unit.id, "_test_description_{}".format(num),
"1")
attr.save()
attr.commit()
self.attributes.append(attr)
if num % count:
index += 1
if index >= len(self.sub_themes):
return
sub_theme = self.sub_themes[index]
except Exception as exp:
logger.error(exp)
self.fail()
def get(self) -> [db.Model]:
"""
Fetch Attributes from the database
:param attribute_id: Attribute id
:param subtheme_id: SubTheme id
:return: A list of Attributes with an HTTPstatus code OK (200) or an error message and a the appropriate
HTTPStatus code
"""
args = self.reqpaser.parse_args()
# Fetch by attribute_id
if "attribute_id" in args and "subtheme_id" not in args:
attribute = Attributes.get_by_id(args["attribute_id"])
if not attribute:
return {
"error": "Attribute not found",
"id": args["attribute_id"]
}, HTTPStatus.NOT_FOUND
content = [attribute]
return [attr.json() for attr in content], HTTPStatus.OK
# Fetch by subtheme_id
elif "attribute_id" not in args and "subtheme_id" in args:
attributes = Attributes.get_by_sub_theme_id(args["subtheme_id"])
if not attributes:
return {
"error": "Attributes not found",
"subtheme_id": args["subtheme_id"]
}, HTTPStatus.NOT_FOUND
if isinstance(attributes, Attributes):
content = [attributes]
return [attr.json() for attr in content], HTTPStatus.OK
return [attr.json() for attr in attributes], HTTPStatus.OK
# Fetch all attribute
attributes = Attributes.get_all()
return [attr.json() for attr in attributes], HTTPStatus.OK
def get(self) -> (dict, HTTPStatus):
"""
Check for triggered alerts.
:return: On success, an HTTP response with a JSON body content
containing the maximum and minimum alerts that have been exceeded with
an HTTP status code of 200 (OK), otherwise an HTTP response with a JSON
body content containing an appropriate error message and appropriate an
HTTP status code
"""
args = self.reqparser.parse_args()
# Use current user_id if not user_id was parsed
if "user_id" not in args:
user = Users.find_by_email(get_jwt_identity())
if user:
args["user_id"] = user.id
else:
# Return Error current user id not found
return (dict(error="User id not found for Current user, "
"User session may have timed out"),
HTTPStatus.INTERNAL_SERVER_ERROR)
else:
user = Users.find_by_id(args["user_id"])
if not user:
# Return error Parsed User Id not found
return dict(error="User with id {} not found.".format(
args["user_id"])), HTTPStatus.NOT_FOUND
# Check Attribute exists
if not Attributes.get_by_id(args["attribute_id"]):
# Return Error Attribute ID not found
return dict(error="Attribute id {} not found.".format(
args["attribute_id"])), HTTPStatus.NOT_FOUND
# Get Attribute Max and Minimums
attribute_range = AttributeRange.get_by_attr_id(args["attribute_id"])
if not attribute_range:
# Return eroor Attribute range not found
return (dict(error="Attribute range not found",
**args), HTTPStatus.NOT_FOUND)
# Check Alerts
max_alerts = AlertWidgetModel.get_max_alerts(attribute_range,
user_id=user.id)
min_alerts = AlertWidgetModel.get_min_alerts(attribute_range,
user_id=user.id)
return dict(max=max_alerts, min=min_alerts), 200
def test_min_max_for_all_attributes(self):
"""
Test whether an AttributeRange entry is created for all Attribute
entries
"""
response = self.testing_client.post('/importer_retry', data=dict(
api_id=1), headers=self.access_token_header)
self.assertEqual(response.status_code, 200)
attributes = Attributes.get_all()
attribute_range_entries = AttributeRange.get_all()
attribute_ids = [attr_range.attribute_id for attr_range in
attribute_range_entries]
for attr in attributes:
self.assertIn(attr.id, attribute_ids)
def get(self):
args = self.parser.parse_args()
theme, subtheme = None, None
if "subtheme" in args:
subtheme = args['subtheme']
if subtheme is not None and subtheme != '':
attributes = Attributes.get_by_sub_theme_id(subtheme)
return [a.json() for a in attributes], 200
elif "theme" in args:
theme = args['theme']
if theme != "":
subthemes = SubTheme.get_by_theme_id(theme)
return [a.json() for a in subthemes], 200
if theme is None and subtheme is None:
themes = Theme.get_all()
return [a.json() for a in themes], 200
return {"error": "error occured while processing request"}, 400
def post(self) -> ({str: str}, HTTPStatus):
"""
Update AttrAlias (Attribute Alias) if it exists otherwise create an AttrAlias
:param attribute_id: Parent Attribute id to alias
:param user_id: Owner user id
:param name: Alias for Attribute name
:param table_name: Alias for Attribute table_name
:param unit_id: Id of unit to be used
:param description: Custom user description for Attribute
:return: AttrAlias with an HTTPstatus code OK (200) or an error message and a the appropriate HTTPStatus code
"""
args = self.reqpaser.parse_args()
attribute = Attributes.get_by_id(args["attribute_id"])
if not attribute:
return {"error": "Attribute Not Found."}, HTTPStatus.NOT_FOUND
user = Users.find_by_id(args["user_id"])
if not user:
return {"error": "User Not Found."}, HTTPStatus.NOT_FOUND
alias = AttrAlias.get_by(user_id=args["user_id"], attribute_id=args["attribute_id"])
if alias:
alias.name = args["name"] if "name" in args else alias.name
alias.table_name = args["table_name"] if "table_name" in args else alias.table_name
alias.unit_id = args["unit_id"] if "unit_id" in args else alias.unit_id
alias.description = args["description"] if "description" in args else alias.description
else:
alias = AttrAlias(args["attribute_id"], args["user_id"],
name=args.get("name"),
table_name=args.get("table_name"),
description=args.get("description"))
try:
alias.save()
alias.commit()
except Exception as e:
logger.error("failed to persist attribute alias", e)
return {"error": "Failed to commit attribute alias to database",
"exception": e}, HTTPStatus.INTERNAL_SERVER_ERROR
return alias.json(), HTTPStatus.OK
def get(self) -> ({str: str}, HTTPStatus):
"""
Fetch AttrAlias (Attribute Alias) from the database
:param attribute_id: Parent Attribute id
:param user_id: Owner user id
:return: AttrAlias with an HTTPstatus code OK (200) or an error message and a the appropriate HTTPStatus code
"""
args = self.reqpaser.parse_args()
attribute = Attributes.get_by_id(args["attribute_id"])
if not attribute:
return {"error": "Attribute Not Found."}, HTTPStatus.NOT_FOUND
user = Users.find_by_id(args["user_id"])
if not user:
return {"error": "User Not Found."}, HTTPStatus.NOT_FOUND
alias = AttrAlias.get_by(user_id=args["user_id"], attribute_id=args["attribute_id"])
if not alias:
return attribute.json(), HTTPStatus.OK
return alias.json(), HTTPStatus.OK
def post(self) -> ({str: str}, HTTPStatus):
"""
Update Attributes SubTheme
:param attribute_id: Attributes identification number
:param sub_theme_id: SubTheme identification number
:type attribute_id: str
:type sub_theme_id: int
:return: A JSON containing a message, Attribute id, SubTheme id and a HTTPStatus 200 (OK) on success
otherwise a JSON with a error message and a HTTPStatus 404 (NotFound)
"""
args = self.reqpaser.parse_args()
attribute = AttrAlias.get_by_attr_id(args["attribute_id"])
if not attribute:
attribute = Attributes.get_by_id(args["attribute_id"])
if not attribute:
return {
"error": "Attribute not found",
"id": args["attribute_id"]
}, HTTPStatus.NOT_FOUND
sub_theme = SubTheme.get_by(id=args["sub_theme_id"])
if not sub_theme:
return {
"error": "SubTheme not found",
"id": args["sub_theme_id"]
}, HTTPStatus.NOT_FOUND
attribute.sub_theme_id(sub_theme.id)
attribute.save()
attribute.commit()
return {
"message": "Attribute SubTheme updated",
"attribute_id": args["attribute_id"],
"sub_theme_id": args["sub_theme_id"]
}, HTTPStatus.OK
def get(self):
args = self.parser.parse_args()
theme, subtheme, attribute_data, sensor, sensor_name, sensor_attribute, attributes, sensorid, n_predictions, predictions, grouped, harmonising_method, per_sensor, freq = None, None, None, None, None, None, [], None, 100, None, None, None, None, '1H'
if 'theme' in args:
theme = args['theme']
if 'subtheme' in args:
subtheme = args['subtheme']
if 'attributedata' in args:
attribute_data = args['attributedata']
if 'attribute' in args and args['attribute'] is not None:
_attributes = args['attribute']
if _attributes != '':
attributes = _attributes.split(',')
if 'sensor' in args and args['sensor'] is not None:
sensor = args['sensor']
if sensor != '':
if sensor == 'all':
sensors = Sensor.get_all()
return [a.json() for a in sensors], 200
else:
return (Sensor.get_by_id(sensor)).json(), 200
if 'sensorname' in args and args['sensorname'] is not None:
sensor_name = args['sensorname']
if sensor_name != '':
_sensors = sensor_name.split(',')
_by_name = Sensor.get_by_name_in(_sensors)
return [a.json() for a in _by_name], 200
if 'sensorattribute' in args and args['sensorattribute'] is not None:
sensor_attribute = args['sensorattribute']
if sensor_attribute != '':
_sen_attrs_ids = sensor_attribute.split(',')
_sen_attrs = SensorAttribute.get_by_id_in(_sen_attrs_ids)
attrs_ids = [_id.a_id for _id in _sen_attrs]
_attributes = Attributes.get_by_id_in(attrs_ids)
return [a.json() for a in _attributes], 200
if 'grouped' in args:
grouped = args['grouped']
if 'harmonising_method' in args:
harmonising_method = args['harmonising_method']
if 'per_sensor' in args:
per_sensor = args['per_sensor']
if 'freq' in args:
freq = args['freq']
if 'predictions' in args:
predictions = args['predictions']
if predictions >=100:
predictions = 100
if 'n_predictions' in args:
n_predictions = args['n_predictions']
if 'sensorid' in args:
sensorid = args['sensorid']
if theme is None and subtheme is None \
and len(attributes) == 0 and attribute_data is None \
and sensor is None and sensor_name is None and sensor_attribute is None:
themes = Theme.get_all()
return [a.json() for a in themes], 200
if attribute_data is not None:
global LIMIT, OFFSET
data = None
operation = None
if 'limit' in args and args['limit'] is not None:
LIMIT = args['limit']
if 'offset' in args and args['offset'] is not None:
OFFSET = args['offset']
if 'operation' in args and args['operation'] is not None:
operation = args['operation']
if ('fromdate' in args and args['fromdate'] is not None
and 'todate' in args and args['todate'] is not None):
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET,
args['fromdate'], args['todate'], operation)
if predictions:
data.append(self.get_predictions(attribute_table = data[0]["Attribute_Table"],
sensor_id = sensorid,
n_pred = n_predictions))
else:
if grouped:
if harmonising_method:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
data = request_harmonised_data(data, harmonising_method=harmonising_method)
else:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
data = request_grouped_data(data, per_sensor=per_sensor, freq=freq)
else:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
if predictions:
#### Ceck for data
if data[0]["Total_Records"] != 0:
#### Check for non numeric data
if is_number(data[0]["Attribute_Values"][0]["Value"]):
data.append(self.get_predictions(attribute_table = data[0]["Attribute_Table"],
sensor_id = sensorid,
n_pred = n_predictions))
else:
print("Cannot predict non-numeric data")
pass
else:
pass
return data, 200
if attributes:
_attrs = []
attr = Attributes.get_by_name_in(attributes)
for a in attr:
_attrs.append(a.json())
return _attrs, 200
if subtheme is not None and subtheme != '':
attributes = Attributes.get_by_sub_theme_id(subtheme)
return [a.json() for a in attributes], 200
if theme is not None and theme != '':
subthemes = SubTheme.get_by_theme_id(theme)
return [a.json() for a in subthemes], 200
return {
"error": "error occured while processing request"
}, 400
def get_attribute_data(self, attribute_name, limit, offset,
fromdate=None, todate=None, operation=None):
# clearing previous metadata
db.metadata.clear()
attrs = attribute_name.split(',')
attributes = Attributes.get_by_name_in(attrs)
data = []
for attribute in attributes:
model = ModelClass(attribute.table_name.lower())
count = db.session.query(model).count()
values = []
if fromdate is not None and todate is not None:
if operation is None:
values = db.session.query(model) \
.filter(model.api_timestamp >= fromdate) \
.filter(model.api_timestamp <= todate) \
.limit(limit).offset(abs(count - offset)) \
.all()
else:
values = db.session.query(model) \
.filter(model.api_timestamp >= fromdate) \
.filter(model.api_timestamp <= todate) \
.all()
else:
if operation is None:
### refactored the query to fetch the latest values by default
values = db.session.query(model).order_by(desc(model.api_timestamp)).limit(limit).all() # \
# values = db.session.query(model).limit(limit) \
# .offset(abs(count - offset)).all()
else:
values = db.session.query(model).all()
_common = {
'Attribute_Table': attribute.table_name,
'Attribute_Name': attribute.name,
'Attribute_Description': attribute.description,
'Attribute_Unit_Value': attribute.unit_value,
'Total_Records': count
}
temp = []
if operation is None:
for i in range(len(values)-1, -1, -1):
temp.append({
'Sensor_id': values[i].s_id,
'Value': values[i].value,
'Timestamp': str(values[i].api_timestamp)
})
_common['Attribute_Values'] = temp
else:
_values = [v.value for v in values]
_int_values = list(map(float, _values))
_operation_result = 0
if operation == 'sum':
_operation_result = sum(_int_values)
elif operation == 'mean':
_operation_result = sum(_int_values) / len(_int_values)
elif operation == 'median':
_operation_result = statistics.median(_int_values)
_common['Result_' + operation] = _operation_result
data.append(_common)
return data
def get(self):
args = self.parser.parse_args()
attribute_data, attributes, sensorid, n_predictions, predictions, grouped, harmonising_method, per_sensor, freq = None, [], None, 100, None, None, None, None, '1H'
if 'attributedata' in args:
attribute_data = args['attributedata']
if 'attribute' in args and args['attribute'] is not None:
_attributes = args['attribute']
if _attributes != '':
attributes = _attributes.split(',')
if 'grouped' in args:
grouped = args['grouped']
if 'harmonising_method' in args:
harmonising_method = args['harmonising_method']
if 'per_sensor' in args:
per_sensor = args['per_sensor']
if 'freq' in args:
freq = args['freq']
if 'predictions' in args:
predictions = args['predictions']
if predictions >=100:
predictions = 100
if 'n_predictions' in args:
n_predictions = args['n_predictions']
if 'sensorid' in args:
sensorid = args['sensorid']
if attribute_data is not None:
global LIMIT, OFFSET
data = None
operation = None
if 'limit' in args and args['limit'] is not None:
LIMIT = args['limit']
if 'offset' in args and args['offset'] is not None:
OFFSET = args['offset']
if 'operation' in args and args['operation'] is not None:
operation = args['operation']
if ('fromdate' in args and args['fromdate'] is not None
and 'todate' in args and args['todate'] is not None):
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET,
args['fromdate'], args['todate'], operation)
if predictions:
data.append(self.get_predictions(attribute_table = data[0]["Attribute_Table"],
sensor_id = sensorid,
n_pred = n_predictions))
else:
if grouped:
if harmonising_method:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
data = request_harmonised_data(data, harmonising_method=harmonising_method)
else:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
data = request_grouped_data(data, per_sensor=per_sensor, freq=freq)
else:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
if predictions:
#### Ceck for data
if data[0]["Total_Records"] != 0:
#### Check for non numeric data
if is_number(data[0]["Attribute_Values"][0]["Value"]):
data.append(self.get_predictions(attribute_table = data[0]["Attribute_Table"],
sensor_id = sensorid,
n_pred = n_predictions))
else:
print("Cannot predict non-numeric data")
pass
else:
pass
return data, 200
if attributes:
_attrs = []
attr = Attributes.get_by_name_in(attributes)
for a in attr:
_attrs.append(a.json())
return _attrs, 200
return {
"error": "error occured while processing request"
}, 400
class TestAttrAlias(TestCase):
"""
Unittest Attribute Alias, Test Endpoints and Database persistence.
"""
def setUp(self):
"""
Setup a FlaskClient, AppContext, Admin user, Authorization header for requests to
the Flask Client and a dummy Theme
"""
self.client, self.app_context = self.create_test_client()
self.user = self.create_admin_user()
self.auth_header = self.get_auth_header()
self.unit = Unit("_test_unit_type_", "A test unit")
self.unit.save()
self.unit.commit()
self.attribute = Attributes.get_by_name("_test_attribute_")
self.theme = self.create_dummy_theme()
self.subtheme = self.create_dummy_subtheme()
if not self.attribute:
self.attribute = Attributes("1234567890-123456789-123456789", "_test_attribute_", "_table_name_",
self.subtheme.id, self.unit.id)
self.attribute.save()
self.attribute.commit()
self.clean_up = [self.user, self.attribute, self.theme, self.subtheme, self.theme, self.unit]
def create_test_client(self) -> (FlaskClient, AppContext):
"""
Create FlaskClient and AppContext
:return: FlaskClient and AppContext
"""
test_app = create_app(DATABASE_NAME='test_analysis', TESTING=True)
testing_client = test_app.test_client()
test_app_context = test_app.app_context()
test_app_context.push()
return testing_client, test_app_context
def create_dummy_theme(self) -> Theme:
"""
Create a Theme
:return: a Theme
"""
theme = Theme.get_by_name("_test_add_theme_")
if not theme:
theme = Theme("_test_add_theme_")
theme.save()
theme.commit()
return theme
return theme
def create_dummy_subtheme(self) -> SubTheme:
"""
Create SubTheme
:return: SubTheme
"""
subtheme = SubTheme.get_by_name('_TEST_SUB_THEME_')
if not subtheme:
subtheme = SubTheme(self.theme.id, '_TEST_SUB_THEME_')
subtheme.save()
subtheme.commit()
subtheme = SubTheme.get_by_name('_TEST_SUB_THEME_')
return subtheme
def create_admin_user(self) -> Users:
"""
Create Admin user
:return: an admin user
"""
password_hash = bcrypt.hashpw("wfnbqk".encode("utf-8"), bcrypt.gensalt())
user = Users.find_by_email("*****@*****.**")
if not user:
user = Users("Admin", "*****@*****.**", password_hash.decode("utf8"), True, True)
try:
user.save()
user.commit()
except Exception as e:
pass
return user
def get_auth_header(self) -> {str: str}:
"""
Create an Authorization header
:return: An authorization header
"""
response_login = self.client.post('/login', data=dict(email=self.user.email, password="******", remember=True),
follow_redirects=True)
response_login_json = response_login.get_json()
return {'Authorization': 'Bearer {}'.format(response_login_json["access_token"])}
def get_dummy_alias(self) -> db.Model:
"""
Create dummy AttrAlias
:return: A AttrAlias
"""
alias = AttrAlias.get_by(name="_custom_name")
if not alias:
alias = AttrAlias(self.attribute.id, self.user.id, name="_custom_name", table_name="_table_name_",
description="a custon description")
alias.save()
alias.commit()
self.clean_up.append(alias)
return alias
def test_create_alias(self) -> None:
"""
Create AttrAlias database entry using database session
"""
alias = AttrAlias(self.attribute.id, self.user.id, name="_custom_name", table_name="_table_name_",
description="a custon description")
self.assertTrue(alias, msg="test_create_alias(): Failed to create Alias")
if alias:
self.clean_up.append(alias)
def test_create_alias(self) -> None:
""" Create AttrAlias using endpoint and check client response for https status code 200"""
json_payload = {"user_id": self.user.id, "attribute_id": self.attribute.id, "name": "A_Custom_Name_12890",
"table_name": "_A_TaBlE_NaMe_"}
response = self.client.post('/admin/attributes/alias', json=json_payload, headers=self.auth_header)
if response.status_code == HTTPStatus.OK:
alias = AttrAlias.get_by(user_id=self.user.id, name="A_Custom_Name_12890")
if alias:
self.clean_up.append(alias)
self.assertEqual(response.status_code, HTTPStatus.OK)
def test_update_alias(self) -> None:
""" Update AttrAlias using endpoint, Check client response for https status code 200 and update table_name"""
alias = self.get_dummy_alias()
json_payload = {"user_id": self.user.id, "attribute_id": self.attribute.id, "name": "A_Custom_Name_12890",
"table_name": "_UpDaTeD_TaBlE_NaMe_"}
response = self.client.post('/admin/attributes/alias', json=json_payload, headers=self.auth_header)
if response.status_code == HTTPStatus.OK:
alias = AttrAlias.get_by(user_id=self.user.id, name="A_Custom_Name_12890")
if alias:
self.clean_up.append(alias)
self.assertEqual(response.status_code, HTTPStatus.OK)
self.assertEqual(alias.table_name, "_UpDaTeD_TaBlE_NaMe_")
def test_delete_alias(self) -> None:
"""Delete AttrAlias using endpoint and check client response for https status code 204"""
alias = self.get_dummy_alias()
self.assertTrue(alias)
json_payload = {"user_id": self.user.id, "attribute_id": self.attribute.id}
response = self.client.post('/admin/attributes/delete_alias', json=json_payload, headers=self.auth_header)
self.assertEqual(response.status_code, HTTPStatus.NO_CONTENT)
def delete(self, module: db.Model) -> None:
module.delete()
module.commit()
def tearDown(self) -> None:
"""
Clean up dependencies
"""
self.subtheme.delete()
self.subtheme.commit()
self.theme.delete()
self.theme.commit()
db.engine.execute("DELETE FROM attributes WHERE id = 'A_TEST_ATTRIBUTE_ID_'")
# self.delete(self.attribute)
self.unit.delete()
self.unit.commit()
self.user.delete()
self.user.commit()
map(self.delete,self.clean_up)
self.app_context.pop()
class TestGetAttributes(TestCase):
""" Test get_attribute endpoint """
def setUp(self) -> None:
"""
Setup FlaskClient for tests, create an admin user and create the authorization header for requests to
the FlaskClient
"""
self.clean_up = []
self.client, self.app_context = self.create_test_client()
self.user = self.create_admin_user()
self.theme = self.create_dummy_theme()
self.subtheme = self.create_dummy_subtheme()
self.unit = self.get_dummy_unit()
self.attribute = Attributes("A_TEST_ATTRIBUTE_ID_",
"_TEST_ATTR_NAME_",
"_A_TABLE_NAME",
self.subtheme.id,
self.unit.id,
description="_A_CUSTOM_DESCRIPTION_")
self.attribute.save()
self.attribute.commit()
self.clean_up.append(self.attribute)
def create_test_client(self) -> (FlaskClient, AppContext):
"""
Create FlaskClient and AppContext
:return: FlaskClient and AppContext
"""
test_app = create_app(DATABASE_NAME='test_analysis', TESTING=True)
testing_client = test_app.test_client()
test_app_context = test_app.app_context()
test_app_context.push()
return testing_client, test_app_context
def create_dummy_theme(self) -> Theme:
"""
Create a Theme
:return: a Theme
"""
theme = Theme.get_by_name("_test_add_theme_")
if not theme:
theme = Theme("_test_add_theme_")
theme.save()
theme.commit()
self.clean_up.append(theme)
return theme
return theme
def create_dummy_subtheme(self) -> SubTheme:
"""
Create SubTheme
:return: SubTheme
"""
subtheme = SubTheme.get_by_name('_TEST_SUB_THEME_')
if not subtheme:
subtheme = SubTheme(self.theme.id, '_TEST_SUB_THEME_')
subtheme.save()
subtheme.commit()
self.clean_up.append(subtheme)
subtheme = SubTheme.get_by_name('_TEST_SUB_THEME_')
return subtheme
def get_dummy_unit(self):
unit = Unit.get_by_symbol("_A_UNIT_FOR_TESTING")
if not unit:
unit = Unit("_A_UNIT_FOR_TESTING", "_A_TEST_UNIT_DESCRIPTION_")
unit.save()
unit.commit()
if unit not in self.clean_up:
self.clean_up.append(unit)
return unit
def create_admin_user(self) -> Users:
"""
Create Admin user
:return: an admin user
"""
password_hash = bcrypt.hashpw("wfnbqk".encode("utf-8"),
bcrypt.gensalt())
user = Users.find_by_email("*****@*****.**")
if not user:
user = Users("Admin", "*****@*****.**",
password_hash.decode("utf8"), True, True)
try:
user.save()
user.commit()
except Exception as e:
pass
self.clean_up.append(user)
return user
def get_auth_header(self) -> {str: str}:
"""
Create an Authorization header
:return: An authorization header
"""
response_login = self.client.post('/login',
data=dict(email=self.user.email,
password="******",
remember=True),
follow_redirects=True)
response_login_json = response_login.get_json()
return {
'Authorization':
'Bearer {}'.format(response_login_json["access_token"])
}
def test_get_all_attributes(self) -> None:
""" Fetch all Attributes using endpoint"""
response = self.client.get('/admin/attributes/get_attributes')
self.assertEqual(response.status_code, HTTPStatus.OK)
def test_get_attribute_by_id(self) -> None:
""" Fetch Attribute by id using endpoint"""
query_string_data = {"attribute_id": self.attribute.id}
response = self.client.get('/admin/attributes/get_attributes',
data=query_string_data)
self.assertEqual(response.status_code, HTTPStatus.OK)
json_response = response.get_json()
self.assertEqual(json_response[0]["id"],
self.attribute.id,
msg="Attribute id does not match")
self.assertEqual(json_response[0]["name"],
self.attribute.name,
msg="Attribute name does not match")
def delete(self, module: db.Model) -> None:
"""
Delete db>Model instances from the database
:param module: Model to be deleted
:type module: db>Model
"""
module.delete()
module.commit()
def tearDown(self) -> None:
"""
Clean up dependencies
"""
self.subtheme.delete()
self.subtheme.commit()
self.theme.delete()
self.theme.commit()
db.engine.execute(
"DELETE FROM attributes WHERE id = 'A_TEST_ATTRIBUTE_ID_'")
# self.delete(self.attribute)
self.unit.delete()
self.unit.commit()
self.user.delete()
self.user.commit()
map(self.delete, self.clean_up)
self.app_context.pop()
def range_wrapper(*args: Any, **kwargs: dict):
"""
Execute importer function and then compute minimum and maximum
values of attributes
:param args: Arguments of import_function parameter
:param kwargs: Keyword Arguments of import_function parameter
"""
import_function(*args, **kwargs)
attribute_entries = Attributes.get_all()
for attribute in attribute_entries:
attribute_range = AttributeRange.get_by_attr_id(attribute.id)
if attribute_range:
most_recent_entry = Attributes.most_recent_timestamp(
attribute.table_name)
if most_recent_entry:
if attribute_range.latest_update < most_recent_entry:
attr_min = Attributes.attribute_min(
attribute.table_name)
attr_max = Attributes.attribute_max(
attribute.table_name)
try:
attribute_range.minimum_sensor_id = \
attr_min.s_id
attribute_range.minimum = attr_min.value
attribute_range.minimum_recorded_date = \
attr_min.timestamp
attribute_range.maximum_sensor_id = \
attr_max.s_id
attribute_range.maximum = attr_max.value
attribute_range.maximum_recorded_date = \
attr_max.timestamp
attribute_range.latest_update = datetime.now()
attribute_range.save()
attribute_range.commit()
PushAlert.check_alerts(attribute_range)
check_min_and_max_alert_widgets(attribute_range)
except AttributeError:
pass
else:
attr_min = Attributes.attribute_min(attribute.table_name)
attr_max = Attributes.attribute_max(attribute.table_name)
try:
new_range_entry = \
AttributeRange(attribute.id,
attr_min.s_id,
attr_min.value,
attr_min.timestamp,
attr_max.s_id,
attr_max.value,
attr_max.timestamp,
datetime.now())
new_range_entry.save()
new_range_entry.commit()
check_min_and_max_alert_widgets(new_range_entry)
PushAlert.check_alerts(new_range_entry)
except AttributeError:
new_range_entry = AttributeRange(attribute.id, None, None,
None, None, None, None,
datetime.now())
new_range_entry.save()
new_range_entry.commit()
def check_min_and_max_alert_widgets(attribute_range_entry: db.Model):
"""
Send emails to users for alert widgets that have been triggered
:param attribute_range_entry: Entry in the attribute range table
"""
if attribute_range_entry.maximum:
max_alerts = AlertWidgetModel.get_max_alerts(attribute_range_entry)
for alert in max_alerts:
user_details = Users.find_by_id(alert["user_id"])
if user_details:
attr = Attributes.get_by_id(attribute_range_entry.attribute_id)
if attr:
if not send_alert_email(
user_details.email, user_details.fullname,
attr.name, attribute_range_entry.maximum,
attribute_range_entry.maximum_recorded_date,
attribute_range_entry.maximum_sensor_id,
alert["max_threshold"], "exceeded"):
logger.error("Server error prevented the "
"sending of a max alert email "
"to {} regarding attribute with "
"id {}".format(
user_details.email,
attribute_range_entry.attribute_id))
else:
logger.error("Could not send max alert email to "
"user with id {} as the attribute with "
"id {} does not exist ".format(
alert["user_id"],
attribute_range_entry.attribute_id))
else:
logger.error("Could not send max alert email to "
"user with id {} as the user does "
"not exist ".format(alert["user_id"]))
if attribute_range_entry.minimum:
min_alerts = AlertWidgetModel.get_min_alerts(attribute_range_entry)
for alert in min_alerts:
user_details = Users.find_by_id(alert["user_id"])
if user_details:
attr = Attributes.get_by_id(attribute_range_entry.attribute_id)
if attr:
if not send_alert_email(
user_details.email, user_details.fullname,
attr.name, attribute_range_entry.minimum,
attribute_range_entry.minimum_recorded_date,
attribute_range_entry.minimum_sensor_id,
alert["min_threshold"], "fell short of"):
logger.error("Server error prevented the sending of "
"a min alert email to {} regarding "
"attribute with id {}".format(
user_details.email,
attribute_range_entry.attribute_id))
else:
logger.error("Could not send min alert email to "
"user with id {} as the attribute with "
"id {} does not exist ".format(
alert["user_id"],
attribute_range_entry.attribute_id))
else:
logger.error("Could not send min alert email to "
"user with id {} as the user does "
"not exist ".format(alert["user_id"]))