def test_return_attribute_of_subtheme(self):
"""
Adding a theme and subtheme to the database and then testing to see if it's data is
retrieved correctly
"""
theme = Theme("Test_Theme")
theme.save()
theme.commit()
sub_theme = SubTheme(theme.id, "Test_Sub_Theme")
sub_theme.save()
sub_theme.commit()
attributes = Attributes("1234567890-123456789-123456789",
"_test_attribute_", "_table_name_",
sub_theme.id, 1)
attributes.save()
attributes.commit()
response = self.testing_client.get('/data',
data=dict(subtheme=theme.id))
self.assertEqual(theme.json(), response.get_json())
attributes.delete()
attributes.commit()
sub_theme.delete()
sub_theme.commit()
theme.delete()
theme.commit()
def create_subtheme(self, theme_id: str, name: str) -> db.Model:
"""
Create new SubTheme
:param theme_id: Theme id
:param name: SubTheme name
:return: Subtheme
"""
sub_theme = SubTheme(t_id=theme_id, name=name)
sub_theme.save()
return sub_theme
def post(self) -> ({str: str}, HTTPStatus):
"""
Delete an existing Theme.
:param name: name of Theme to delete.
:param id: id of Theme to delete.
:type name: str
:type id: str
:returns: A no content with a http status code of 204, otherwise a JSON of the error details
and the appropriate http status code
"""
if not get_jwt_claims()['admin']:
return {"error": "administration privileges required"}, HTTPStatus.FORBIDDEN
# Get arguments
args = self.reqparser.parse_args()
# does the theme exist?
theme = Theme.get_by_name(args["name"]) if "name" in args else Theme.get_by_id(args["id"])
if not theme:
# cannot delete a theme that does not exist.
return {'error': 'Theme does not exists.', 'id': " ", 'name': args["name"]}, HTTPStatus.BAD_REQUEST
sub_themes = SubTheme.get_by_theme_id(theme.id)
for sub_theme in sub_themes:
sub_theme.delete()
sub_theme.commit()
# delete the theme
theme.delete()
theme.commit()
return "", HTTPStatus.NO_CONTENT
def create_dummy_subtheme(self) -> SubTheme:
"""
Create SubTheme
:return: SubTheme
"""
subtheme = SubTheme.get_by_name('_TEST_SUB_THEME_')
if not subtheme:
subtheme = SubTheme(self.theme.id, '_TEST_SUB_THEME_')
subtheme.save()
subtheme.commit()
subtheme = SubTheme.get_by_name('_TEST_SUB_THEME_')
return subtheme
def tearDown(self) -> None:
""" Clean up all dependencies after tests"""
for alias in self.aliases:
try:
if AttrAlias.get_by_user_id(self.user.id):
alias.delete()
alias.commit()
except Exception:
pass
for attr in self.attributes:
try:
if Attributes.get_by_id(attr.id):
attr.delete()
attr.commit()
except Exception:
pass
for sub_theme in self.sub_themes:
try:
if SubTheme.get_by_id(sub_theme.id):
sub_theme.delete()
sub_theme.commit()
except Exception:
pass
for unit in self.units:
try:
if Unit.get_by_id(unit.id):
unit.delete()
unit.commit()
except Exception:
pass
try:
if Theme.get_by_id(self.theme.id):
self.theme.delete()
self.theme.commit()
except Exception:
pass
self.client.post('/logout', headers=self.auth_header)
if self.user:
if Users.find_by_id(self.user.id):
try:
self.user.delete()
self.user.commit()
except Exception:
pass
self.app_context.pop()
def post(self) -> ({str: str}, HTTPStatus):
"""
Delete an existing SubTheme.
:param name: the name of SubTheme.
:param id: id of SubTheme.
:param theme_id: Parent theme id.
:type name: str
:type id: str
:type theme_id: str
:returns: A no content with a http status code of 204, otherwise a JSON of the error details
and the appropriate http status code
"""
if not get_jwt_claims()['admin']:
return {"error": "administration privileges required"}, HTTPStatus.FORBIDDEN
# Get arguments
args = self.reqparser.parse_args()
subtheme = None
# does the theme exist?
if "id" in args:
subtheme = SubTheme.get_by(id=args["id"])
elif "name" in args and "theme_id" in args:
subtheme = SubTheme.get_by(name=args["name"], t_id=args["theme_id"])
if not subtheme:
if not ("name" in args and "theme_id" in args):
return ({"error": "Argument dependency error: deletion by name requires theme_id"},
HTTPStatus.BAD_REQUEST)
# cannot delete a theme that does not exist.
return {'error': 'Sub-theme does not exists.'}, HTTPStatus.NOT_FOUND
# delete the theme
subtheme.delete()
subtheme.commit()
return "", HTTPStatus.NO_CONTENT
def create_sub_theme(theme: Theme, name: str = 'Airquality') -> None:
"""
Create a SubTheme
:param theme: Parent Theme
:param name: SubThemes name
:raises ValueError: If the new SubTheme is not persisted to the dB
"""
sub_theme = SubTheme.get_by_name(name)
if not sub_theme:
sub_theme = SubTheme(theme.id, name)
sub_theme.save()
sub_theme.commit()
if not sub_theme:
logger.critical('ValueError raised while creating SubTheme')
raise ValueError
def test_submit():
u = Unit('kg', 'Kilogram')
u2 = Unit('g', 'Grams')
u3 = Unit('km', 'KiloMeter')
u.save()
u2.save()
u3.save()
t = Theme('Environment')
t2 = Theme('Transport')
t.save()
t2.save()
st = SubTheme(t.id, 'Airquality')
st2 = SubTheme(t2.id, 'Traffic')
st.save()
st2.save()
db.session.commit()
def get(self):
args = self.parser.parse_args()
theme, subtheme = None, None
if "subtheme" in args:
subtheme = args['subtheme']
if subtheme is not None and subtheme != '':
attributes = Attributes.get_by_sub_theme_id(subtheme)
return [a.json() for a in attributes], 200
elif "theme" in args:
theme = args['theme']
if theme != "":
subthemes = SubTheme.get_by_theme_id(theme)
return [a.json() for a in subthemes], 200
if theme is None and subtheme is None:
themes = Theme.get_all()
return [a.json() for a in themes], 200
return {"error": "error occured while processing request"}, 400
def post(self) -> ({str: str}, HTTPStatus):
"""
Update Attributes SubTheme
:param attribute_id: Attributes identification number
:param sub_theme_id: SubTheme identification number
:type attribute_id: str
:type sub_theme_id: int
:return: A JSON containing a message, Attribute id, SubTheme id and a HTTPStatus 200 (OK) on success
otherwise a JSON with a error message and a HTTPStatus 404 (NotFound)
"""
args = self.reqpaser.parse_args()
attribute = AttrAlias.get_by_attr_id(args["attribute_id"])
if not attribute:
attribute = Attributes.get_by_id(args["attribute_id"])
if not attribute:
return {
"error": "Attribute not found",
"id": args["attribute_id"]
}, HTTPStatus.NOT_FOUND
sub_theme = SubTheme.get_by(id=args["sub_theme_id"])
if not sub_theme:
return {
"error": "SubTheme not found",
"id": args["sub_theme_id"]
}, HTTPStatus.NOT_FOUND
attribute.sub_theme_id(sub_theme.id)
attribute.save()
attribute.commit()
return {
"message": "Attribute SubTheme updated",
"attribute_id": args["attribute_id"],
"sub_theme_id": args["sub_theme_id"]
}, HTTPStatus.OK
def create_theme_tree(self, theme_id: int, user_id: int) -> None:
"""
Create Theme Tree
:param theme_id: Theme Id
:param user_id: User Id
"""
theme = Theme.get_by_id(theme_id)
if not theme:
# Theme does not exist
self.get_all_themes(user_id)
return
# Create Theme Trunk
theme_tree = theme.serializable
sub_themes = SubTheme.get_by_theme_id(theme_id)
if not sub_themes:
# No SubThemes in Theme return Trunk
return theme_tree
sub_theme_ids = {sub.id for sub in sub_themes}
sub_list = []
for sub in sub_themes:
sub_list.append(sub.serializable)
attribute_by_sub_id = self.get_attributes(user_id, sub_theme_ids,
theme_id)
for sub in sub_list:
# Add Attribute branches
attr = attribute_by_sub_id.get(sub["id"])
if attr:
sub["attributes"] = attr
# Add SubTheme branches
theme_tree["sub_themes"] = sub_list
self.response.append(theme_tree)
def merge_subthemes(self, attribute_data, base_importer):
# Fetch sub themes from database
self._print_in_middle('\nSubTheme available in Database')
sub_themes = SubTheme.get_all()
# Show Sub Themes
for theme in range(len(sub_themes)):
print(str(theme) + '.', sub_themes[theme].name)
self._questions('Does the listed SubThemes contains all the respective themes for this dataset: (Y/N) ')
st_available = input()
if st_available == 'y' or st_available == 'Y':
# This is repetition of code from above, repeating 3rd time
self._print_in_middle('\nAttributes and their respective Unit Values')
self._print_attribute_data(attribute_data)
self._questions(Statuses.CHOOSE_SUB_THEME, end='\n')
for theme in range(len(sub_themes)):
print(sub_themes[theme].name + ':', end='')
attr_theme_ids = input().split()
base_importer.join_attr_sub_theme([attribute_data[int(i)] for i in attr_theme_ids],
sub_themes[theme].id)
else:
# Need to give user an option to be able create new sub theme and themes if necessary
pass
def create_sub_themes(self, count: int) -> None:
"""
Create SubThemes
:param count: The number of SubThemes to create.
"""
if self.theme:
theme_id = self.theme.id
for num in range(0, count):
sub_theme = SubTheme(theme_id,
"_test_sub_theme_{}".format(num))
if sub_theme:
try:
sub_theme.save()
sub_theme.commit()
self.sub_themes.append(sub_theme)
except Exception as exp:
logger.error(exp)
self.tearDown()
def post(self) -> ({str: str}, HTTPStatus):
"""
Create new SubTheme
:param theme: the name of the parent theme
:param theme_id: the identification number of the parent theme
:param subtheme: the name of the sub theme
:type theme: str
:type theme_id: str
:type subtheme: str
:returns: A JSON of the new SubTheme with a http status code of 200, otherwise a JSON of the error details
and the appropriate http status code
"""
if not get_jwt_claims()['admin']:
return {
"error": "administration privileges required"
}, HTTPStatus.FORBIDDEN
args = self.reqparser.parse_args()
if "theme" not in args and "theme_id" not in args:
return {
"error": "theme or theme_id required"
}, HTTPStatus.BAD_REQUEST
theme = None
if "theme_id" in args:
theme = Theme.get_by_id(args["theme_id"])
elif "theme" in args:
theme = Theme.get_by_theme(args["theme"])
if not theme:
return ({
"error":
"Theme not found",
"Theme":
args["theme_id"] if args["theme_id"] else args["theme"]
}, HTTPStatus.NOT_FOUND)
sub_theme = SubTheme.get_by(name=args["subtheme"], t_id=theme.id)
# Avoid duplicating sub themes
if sub_theme:
return {
"error": "sub theme already exists",
"theme_id": theme.id,
"sub_theme_id": sub_theme.id,
"Theme": theme.name,
"subtheme": sub_theme.name
}, HTTPStatus.BAD_REQUEST
sub_theme = SubTheme(theme.id, args["subtheme"])
sub_theme.save()
sub_theme.commit()
return {
"message": "sub theme created",
"theme_id": theme.id,
"sub_theme_id": sub_theme.id,
"Theme": theme.name,
"subtheme": sub_theme.name
}, HTTPStatus.OK
def post(self) -> ({str: str}, HTTPStatus):
"""
Rename an existing SubTheme
:param current_name: the name of the sub theme to rename
:param new_name: the new name for the sub theme
:param theme_id: Parent Theme id
:param id: SubTheme id
:type current_name: str
:type new_name: str
:type theme_id: str
:type id: str
:returns: A JSON of the changes made to the sub theme with a http status code of 200, otherwise
a JSON of the error details and the appropriate http status code
"""
if not get_jwt_claims()['admin']:
return {
"error": "administration privileges required"
}, HTTPStatus.FORBIDDEN
# Get arguments
args = self.reqparser.parse_args()
# Check the current theme name and the new theme name is not empty, abort if it is empty
if "theme_id" not in args and "id" not in args:
return ({
'error':
'Argument dependency: theme_id or id required to rename a SubTheme',
"args": args
}, HTTPStatus.BAD_REQUEST)
subtheme = None
if "id" in args:
subtheme = SubTheme.get_by(id=args["id"])
elif "theme_id" in args:
subtheme = SubTheme.get_by(name=args["current_name"],
t_id=args["theme_id"])
if not subtheme:
# cannot rename a subtheme that does not exist.
return {
'error': 'Sub-theme does not exists.',
'args': args
}, HTTPStatus.NOT_FOUND
# Does the new name for theme exist?
if SubTheme.get_by(name=args["new_name"], t_id=subtheme.t_id):
return {
'error':
'Cannot rename sub-theme to {} ; Sub-theme {} already exists.'.
format(args["new_name"], args["new_name"]),
'id':
"",
'name':
args["current_name"]
}, HTTPStatus.BAD_REQUEST
# rename the new theme
subtheme.name = args["new_name"]
subtheme.save()
subtheme.commit()
return ({
"message": "Subtheme renamed",
"id": subtheme.id,
"old_name": args["current_name"],
"new_name": subtheme.name
}, HTTPStatus.OK)
def post(self) -> (dict, HTTPStatus):
"""
Get dummy data from CSV. Store dummy data in database
:param file_name: File name to extract data from.
:return: A Status Report detailing the dB Entries created and an HTTP
Status code 200 on success otherwise, a JSON error message is returned
with the appropriate HTTPStatus code
"""
args = self.reqparser.parse_args()
# Get size of tables before import
self.loc_stats["before"] = db.session.query(func.count(
LocationData.id)).scalar()
self.tracker_stats["before"] = db.session.query(func.count(
Tracker.id)).scalar()
# Fetch Data from CSV
try:
df = pd.read_csv(args['file_name'])
except IOError as ioe:
logger.error("Unable to parse CSV data to dataframe",
ioe.with_traceback(ioe.__traceback__))
return dict(error="Unable to parse CSV data to dataframe",
trackback=ioe.with_traceback(
ioe.__traceback__)), HTTPStatus.BAD_REQUEST
moving_theme = Theme.get_by_name("Moving_Sensors")
if moving_theme:
moving_sensor_theme_id = moving_theme.id
else:
moving_sensor_theme = Theme("Moving_Sensors")
moving_sensor_theme.save()
moving_sensor_theme.commit()
moving_sensor_theme_id = moving_sensor_theme.id
moving_subtheme = SubTheme.get_by_name("Moving_Airquality")
if moving_subtheme:
moving_sensor_subtheme_id = moving_subtheme.id
else:
moving_sensor_subtheme = SubTheme(moving_sensor_theme_id,
"Moving_Airquality")
moving_sensor_subtheme.save()
moving_sensor_subtheme.commit()
moving_sensor_subtheme_id = moving_sensor_subtheme.id
# Trackers must be unique, Fetch trackers and make dB entries for
# each unique tracker
unique_tracker_ids = df["tracker"].unique()
for tracker_id in unique_tracker_ids:
self.t_ids[self.create_trackers(str(tracker_id),
moving_sensor_subtheme_id)] = 0
# Define Location data Pandas DataFrame Column names
loc_df = df[[
'tracker', 'datetime', 'latitude', 'longitude', 'speed', 'heading',
'elevation', 'charger', 'battery', 'signalquality', 'satcnt'
]]
# Drop all entries that are incomplete have NaN or None/ Null values
loc_df = loc_df.dropna()
# Store Location Data in the dB
for index, row in loc_df.iterrows():
self.add_location_data(row['tracker'], row['datetime'],
row['latitude'], row['longitude'],
row['speed'], row['heading'],
row['elevation'], row['charger'],
row['battery'], row['signalquality'],
row['satcnt'])
self.loc_stats["after"] = db.session.query(func.count(
LocationData.id)).scalar()
self.tracker_stats["after"] = db.session.query(func.count(
Tracker.id)).scalar()
return self.status_report(), 200
def create_subtheme(self, theme_id, name):
sub_theme = SubTheme(t_id=theme_id, name=name)
sub_theme.save()
return sub_theme
def get(self):
args = self.parser.parse_args()
theme, subtheme, attribute_data, sensor, sensor_name, sensor_attribute, attributes, sensorid, n_predictions, predictions, grouped, harmonising_method, per_sensor, freq = None, None, None, None, None, None, [], None, 100, None, None, None, None, '1H'
if 'theme' in args:
theme = args['theme']
if 'subtheme' in args:
subtheme = args['subtheme']
if 'attributedata' in args:
attribute_data = args['attributedata']
if 'attribute' in args and args['attribute'] is not None:
_attributes = args['attribute']
if _attributes != '':
attributes = _attributes.split(',')
if 'sensor' in args and args['sensor'] is not None:
sensor = args['sensor']
if sensor != '':
if sensor == 'all':
sensors = Sensor.get_all()
return [a.json() for a in sensors], 200
else:
return (Sensor.get_by_id(sensor)).json(), 200
if 'sensorname' in args and args['sensorname'] is not None:
sensor_name = args['sensorname']
if sensor_name != '':
_sensors = sensor_name.split(',')
_by_name = Sensor.get_by_name_in(_sensors)
return [a.json() for a in _by_name], 200
if 'sensorattribute' in args and args['sensorattribute'] is not None:
sensor_attribute = args['sensorattribute']
if sensor_attribute != '':
_sen_attrs_ids = sensor_attribute.split(',')
_sen_attrs = SensorAttribute.get_by_id_in(_sen_attrs_ids)
attrs_ids = [_id.a_id for _id in _sen_attrs]
_attributes = Attributes.get_by_id_in(attrs_ids)
return [a.json() for a in _attributes], 200
if 'grouped' in args:
grouped = args['grouped']
if 'harmonising_method' in args:
harmonising_method = args['harmonising_method']
if 'per_sensor' in args:
per_sensor = args['per_sensor']
if 'freq' in args:
freq = args['freq']
if 'predictions' in args:
predictions = args['predictions']
if predictions >=100:
predictions = 100
if 'n_predictions' in args:
n_predictions = args['n_predictions']
if 'sensorid' in args:
sensorid = args['sensorid']
if theme is None and subtheme is None \
and len(attributes) == 0 and attribute_data is None \
and sensor is None and sensor_name is None and sensor_attribute is None:
themes = Theme.get_all()
return [a.json() for a in themes], 200
if attribute_data is not None:
global LIMIT, OFFSET
data = None
operation = None
if 'limit' in args and args['limit'] is not None:
LIMIT = args['limit']
if 'offset' in args and args['offset'] is not None:
OFFSET = args['offset']
if 'operation' in args and args['operation'] is not None:
operation = args['operation']
if ('fromdate' in args and args['fromdate'] is not None
and 'todate' in args and args['todate'] is not None):
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET,
args['fromdate'], args['todate'], operation)
if predictions:
data.append(self.get_predictions(attribute_table = data[0]["Attribute_Table"],
sensor_id = sensorid,
n_pred = n_predictions))
else:
if grouped:
if harmonising_method:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
data = request_harmonised_data(data, harmonising_method=harmonising_method)
else:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
data = request_grouped_data(data, per_sensor=per_sensor, freq=freq)
else:
data = self.get_attribute_data(attribute_data, LIMIT, OFFSET, operation=operation)
if predictions:
#### Ceck for data
if data[0]["Total_Records"] != 0:
#### Check for non numeric data
if is_number(data[0]["Attribute_Values"][0]["Value"]):
data.append(self.get_predictions(attribute_table = data[0]["Attribute_Table"],
sensor_id = sensorid,
n_pred = n_predictions))
else:
print("Cannot predict non-numeric data")
pass
else:
pass
return data, 200
if attributes:
_attrs = []
attr = Attributes.get_by_name_in(attributes)
for a in attr:
_attrs.append(a.json())
return _attrs, 200
if subtheme is not None and subtheme != '':
attributes = Attributes.get_by_sub_theme_id(subtheme)
return [a.json() for a in attributes], 200
if theme is not None and theme != '':
subthemes = SubTheme.get_by_theme_id(theme)
return [a.json() for a in subthemes], 200
return {
"error": "error occured while processing request"
}, 400
def test_return_subthemes_of_theme(self):
"""
Adding a theme and subtheme (linked to to that theme) to the database and then
testing to see if it is retrieved correctly
"""
theme = Theme(name='Test_Theme')
theme.save()
theme.commit()
sub_theme = SubTheme(theme.id, "Test_Sub_Theme")
sub_theme.save()
sub_theme.commit()
response = self.testing_client.get('/data', data=dict(theme=theme.id))
self.assertEqual(sub_theme.json(), response.get_json())
sub_theme.delete()
sub_theme.commit()
theme.delete()
theme.commit()