def insert_dataset(files):
"""
Function for inserting a whole dataset in the database
:param files: dict with FileStorage instances, holding datasets' files
:return: (True, None) | (False, str) - string instance is the error message
"""
# parameters required for basic data such as which dataset to be improted, what time formats to be used, etc.
BASE_PARAMS = ['Date', 'Time']
# parameters required for getting specific columns from given datasets, etc. for Temperature get tempC column
DATASET_PARAMS = ['weatherFormat', 'pollutantFormat']
dataset_metadata = json.load(files['metadata'])
if not isinstance(dataset_metadata, dict):
return False, Errors.WRONG_INSTANCE.value
are_params_missing = Helpers.are_params_missing(
dataset_metadata, BASE_PARAMS + DATASET_PARAMS)
if are_params_missing:
return False, Errors.MISSING_PARAM.value
for x in DATASET_PARAMS:
if not isinstance(dataset_metadata[x], dict):
return False, Errors.WRONG_INSTANCE.value
for key in files:
dataset_metadata[key + 'Datasets'] = files[key]
# Combine multiple datasets and get result
main_transformer = MainTransformer(config=dataset_metadata)
main_transformer.add_transformer(Transformers.WEATHER_TRANSFORMER)
main_transformer.add_transformer(Transformers.POLLUTANT_TRANSFORMER)
main_transformer.transform()
dataset = main_transformer.get_dataset()
result, err = DBManager.insert_dataset(dataset, dataset_metadata)
return result, err
import pandas
import random
import numpy as np
import json
from airpyllution import MainTransformer
from airpyllution import Transformers
from airpyllution import DBManager
with open('configTwo.json') as file:
dataset_one = json.load(file)
with open('configOne.json') as file:
dataset_two = json.load(file)
data_transformer = MainTransformer(config=dataset_one)
data_transformer.add_transformer(Transformers.WEATHER_TRANSFORMER)
data_transformer.add_transformer(Transformers.POLLUTANT_TRANSFORMER)
data_transformer.transform()
dataset_centre = data_transformer.get_dataset()
data_transformer = MainTransformer(config=dataset_two)
data_transformer.add_transformer(Transformers.WEATHER_TRANSFORMER)
data_transformer.add_transformer(Transformers.POLLUTANT_TRANSFORMER)
data_transformer.transform()
dataset_a33 = data_transformer.get_dataset()
length_centre = dataset_centre.shape[0]
length_a33 = dataset_a33.shape[0]
dataset_centre['Longitude'] = -1.463484
def get_single_instance_dataset(body, stats=None, prev=None):
"""
Function for generating a single instance dataset for CNN
:param body: dict - parameters from request
:param stats: dict - stats for dataset normalization on which model was trained
:param prev: int - number of previous records to be generated
:return: DataFrame with predictions
"""
if not isinstance(body, dict):
return None
if 'date_time' not in body or not isinstance(body['date_time'], str):
return None
if 'longitude' not in body or not isinstance(body['longitude'], float):
return None
if 'latitude' not in body or not isinstance(body['latitude'], float):
return None
df_schema = {
'DateTime': [body['date_time']],
'Longitude': body['longitude'],
'Latitude': body['latitude'],
'Pollutant': None
}
instance_object = {
'DateTime': body['date_time'],
'Longitude': body['longitude'],
'Latitude': body['latitude'],
'Pollutant': None
}
data_keys = list()
if 'data' in body and 'weather' in body['data']:
for key in body['data']['weather'].keys():
df_schema[key] = [body['data']['weather'][key]]
instance_object[key] = body['data']['weather'][key]
if isinstance(prev, int):
ready_data = None
DatasetsApi.generate_previous_records(df_schema, prev, ready_data)
dataset = pandas.DataFrame(df_schema)
automatic_normalization = not isinstance(
stats, dict) # if stats parameter is given
dataset.set_index(keys='DateTime', inplace=True)
MainTransformer.periodic_f(dataset)
X_predict, _, _, _, _ = MainTransformer.get_training_and_test_set(
dataset,
'Pollutant',
'Uncertainty',
size=1,
normalize=automatic_normalization)
if not automatic_normalization:
MainTransformer.normalize(X_predict, stats=stats, inplace=True)
return X_predict
def get_dataset(body, use_dataframe=True):
"""
Function for getting a dataset from database
:param body: dict - requires several parameters:
* type - of ML model (CNN, FullGP, etc.),
* range - dict with start and end datetime strings in format Day-Month-Year H:M (24H format)
* locations - list with lists of locations - list of location is a list with longitude and latitude, e.g.
[longitude, latitude]
* pollutant - name of the pollutant, e.g. PM10, PM2.5
* data - dict with additional data such as weather data (data['weather'] is another dict)
:param use_dataframe: bool - whether the returned dataset is a dataframe or a list
:return: DataFrame | List | None
"""
if not isinstance(body, dict):
return None
if 'range' not in body or 'locations' not in body or 'pollutant' not in body:
return None
if body['range'] is None or body['locations'] is None or body[
'pollutant'] is None:
return None
# if not isinstance('range', dict):
# return None
#
# if not isinstance(body['locations'], list):
# return None
# else:
# result = list(filter(lambda c: not isinstance(c, list) or len(c) != 2, body['locations']))
# if len(result) != 0 and len(body['locations']) != 0:
# return None
# Params required for the DBManager, acts as a config of a given dataset
config_params = {
"Date": DatasetsApi.DATE_TIME_FORMAT.split(' ')[0],
"Time": DatasetsApi.DATE_TIME_FORMAT.split(' ')[1],
"pollutant": {
"Pollutant": None
},
'weather': {}
}
start_date = None
end_date = None
uncertainty = False
if 'start' in body['range']:
start_date = datetime.datetime.strptime(
body['range']['start'], DatasetsApi.DATE_TIME_FORMAT)
if 'end' in body['range']:
end_date = datetime.datetime.strptime(body['range']['end'],
DatasetsApi.DATE_TIME_FORMAT)
if 'uncertainty' in body:
uncertainty = True
location_coordinates = []
if isinstance(body['locations'], list):
location_coordinates = list(
map(lambda x: (x[0], x[1]), body['locations']))
if isinstance(body['pollutant'], str):
config_params['pollutant']['Pollutant'] = body['pollutant']
if 'data' in body and isinstance(body['data'], dict):
if 'weather' in body['data'] and isinstance(
body['data']['weather'], dict):
config_params['weather'] = body['data']['weather']
datasets = []
for coordinates_pair in location_coordinates:
dataset, err = DBManager.get_dataset(datetime_from=start_date,
datetime_to=end_date,
longitude=coordinates_pair[0],
latitude=coordinates_pair[1],
config=config_params,
use_dataframe=use_dataframe,
uncertainty=uncertainty)
dataset_size = len(
dataset.index) if use_dataframe else len(dataset)
if err is None and dataset_size != 0:
datasets.append(dataset)
if len(datasets) == 0:
# TODO - IT IS VERY IMPORTANT TO CHANGE ALL CONDITIONS TO CHECK IF df.shape[0] == 0 IN THE API
return pandas.DataFrame() if use_dataframe else []
if use_dataframe:
complete_dataset = pandas.concat(datasets)
MainTransformer.periodic_f(complete_dataset)
else:
complete_dataset = []
for x in datasets:
complete_dataset.extend(x)
return complete_dataset
def make_predictions(body, overwrite=False):
"""
Function for making predictions over a time range and locations by a given model
:param body:
:param overwrite:
:return: bool, list - boolean whether it is successful and list with predictions and uncertanties
"""
dataset = DatasetsApi.get_dataset(body, use_dataframe=True)
if dataset is None:
return False, []
# get dataset with empty pollutant values
incomplete_dataset = dataset if overwrite else dataset[
dataset['Pollutant'].isnull()]
# split the dataset, do not normalize until means and stds are taken from the model
X_predict, y_predict, _, _, stats = MainTransformer.get_training_and_test_set(
incomplete_dataset,
'Pollutant',
'Uncertainty',
size=1,
normalize=False)
model, model_record, err = ModelApi.get_model_by_name(body['name'])
predictions = []
print(err)
if err is None:
training_dataset_stats = {}
print('Verifying features...')
if X_predict is None or X_predict.shape[1] != model.n_features:
print('Wrong number of features')
print(X_predict.shape[1] - 1)
print(model.n_features)
return False, []
print('Checking model stats...')
if 'dataset_stats' in model.stats:
training_dataset_stats = model.stats['dataset_stats']
feature_names = set(training_dataset_stats.keys())
dataset_features = set(X_predict)
dataset_features.discard('DateTime')
print('Checking feature names...')
if feature_names != dataset_features:
return False, []
print('Normalizing...')
MainTransformer.normalize(X_predict,
stats=training_dataset_stats,
inplace=True)
else:
return False, []
print('Preidicting...')
predictions = model.predict(X_predict, uncertainty=True)
MainTransformer.unnormalize(X_predict,
training_dataset_stats,
inplace=True)
MainTransformer.remove_periodic_f(X_predict)
X_predict.loc[:, 'Pollutant'] = Series([x[0] for x in predictions],
index=X_predict.index)
X_predict.loc[:,
'Uncertainty'] = Series([x[1] for x in predictions],
index=X_predict.index)
# add predictions to the DB
print('Done. Adding to database...')
optional_data_keyset = set(body['data'].keys())
dataframe_optional_data = set(X_predict.keys()).difference(
ModelApi.REQUIRED_FIELDS)
keys_with_data_to_be_added = optional_data_keyset.intersection(
dataframe_optional_data)
results = []
for index, row in X_predict.iterrows():
if row['Pollutant'] is not None and math.isnan(
row['Pollutant']):
continue
input_instance = {
'date_time': index,
'longitude': row['Longitude'],
'latitude': row['Latitude'],
'pollutant': body['pollutant'],
'pollution_value': row['Pollutant'],
'uncertainty': row['Uncertainty'],
'data': {}
}
print(body['pollutant'])
print(row['Pollutant'])
for key in keys_with_data_to_be_added:
input_instance['data'][key] = row[key]
result = DatasetsApi.insert_single_prediction(input_instance)
results.append(result)
predictions = ModelApi.__predictions_to_primitive_float(
predictions)
print('failed following: ')
print(list(filter(lambda x: not x[0], results)))
return True, predictions
return False, predictions # in case that model does not exist
def train_model(model_name, body):
"""
Function for further training a model provided that the model already exists in the DB
:param model_name: str - name of the existing model
:param body: dict - body of the request
:return: (True, None) | (False, str) | (False, list)
"""
print('Getting dataset...')
model, model_record, err = ModelApi.get_model_by_name(model_name)
if model is None:
return False, err
dataset = DatasetsApi.get_dataset(body, use_dataframe=True)
if dataset is None:
return False, Errors.NO_DATA.value
complete_dataset = dataset[dataset['Pollutant'].notnull()]
if 'n_instances_trained' in model.stats and 'dataset_stats' in model.stats:
updated_stats, new_stats = MainTransformer.normalize_with_old_stats(
model.stats['n_instances_trained'],
model.stats['dataset_stats'], complete_dataset)
MainTransformer.normalize(complete_dataset,
stats=updated_stats,
inplace=True)
else:
return False, []
stats = new_stats
X_train, y_train, _, _, _ = MainTransformer.get_training_and_test_set(
complete_dataset,
'Pollutant',
'Uncertainty',
size=1,
normalize=False)
training_dataset_stats = {}
print('Verifying dataset...')
if 'dataset_stats' in model.stats:
training_dataset_stats = model.stats['dataset_stats']
feature_names = set(training_dataset_stats.keys())
dataset_features = set(X_train)
dataset_features.discard('DateTime')
print('Verifying dataset features')
if feature_names != dataset_features:
print('feature names', feature_names, training_dataset_stats,
training_dataset_stats.keys())
print('dataset features', dataset_features)
if feature_names.intersection(
dataset_features) == feature_names:
print('Dataset is in the expected shape')
print('difference')
difference = dataset_features.difference(feature_names)
print(difference)
MainTransformer.remove_features(X_train, difference)
else:
print(feature_names)
print(dataset_features)
return False, []
else:
return False, []
print('Starting to train model...')
model.train(X_train, y_train, stats=stats)
model_params, extra_params = model.model_to_json()
result = DBManager.upsert_model(model_name,
model_record.type,
model_record.resource,
model_params=model_params,
extra_params=extra_params)
print(result)
return result
def create_model(name, body):
"""
Function for creating a non-existing model and training it with a given dataset
This function should happen in the background to prevent overhead to Flask
:param name: unique name of the model
:param body: dict with following data:
* type - type of model (CNN, FullGP, etc.)
* range - dict with start and end fields, each storing datetime in DATE_TIME_FORMAT
* locations - list of lists, nested list should have two entries 0 - longitude, 1 - latitude
* pollutant - name of the polllutant PM10, PM2.5
* data - dict object with additional data that would be stored as JSONB data, it could have keys such as
weather
:return: bool: whether model was created
"""
if body is None:
return False, Errors.MISSING_BODY.value
print('Getting dataset...')
dataset = DatasetsApi.get_dataset(body, use_dataframe=True)
print(dataset)
if dataset is None:
return False, Errors.NO_DATA.value
model = None
complete_dataset = dataset[dataset['Pollutant'].notnull()]
X_train, y_train, _, _, stats = MainTransformer.get_training_and_test_set(
complete_dataset,
'Pollutant',
'Uncertainty',
size=1,
normalize=True)
if 'type' not in body:
return False, Errors.NO_MODEL_TYPE_GIVEN.value
if body['type'] == 'CNN':
model = ConvolutionalNeuralNetwork()
model.train(X_train, y_train, stats=stats)
resource = 'keras'
model_params, extra_params = model.model_to_json()
result = DBManager.upsert_model(name,
body['type'],
resource,
model_params=model_params,
extra_params=extra_params)
return True, None
elif body['type'] == 'FullGP':
model = GaussianProcesses()
model.train(X_train, y_train, stats=stats)
resource = 'GPy'
model_params, extra_params = model.model_to_json()
result = DBManager.upsert_model(name,
body['type'],
resource,
model_params=model_params,
extra_params=extra_params)
return True, None
elif body['type'] == 'SparseGP':
model = SparseGaussianProcesses()
model.train(X_train, y_train, stats=stats)
resource = 'GPy'
model_params, extra_params = model.model_to_json()
result = DBManager.upsert_model(name,
body['type'],
resource,
model_params=model_params,
extra_params=extra_params)
return True, None
return False, Errors.NO_SUCH_MODEL_TYPE.value
def make_single_prediction(body):
"""
Function for making predictions over a time range and locations by a given model
:param body:
:return: bool, list - boolean whether it is successful and list with predictions and uncertanties
"""
if not isinstance(body, dict):
return False, []
if 'name' not in body:
return False, []
if 'pollutant' not in body:
return False, []
model, model_record, err = ModelApi.get_model_by_name(body['name'])
predictions = []
if err is None:
prev = None
if isinstance(model, ConvolutionalNeuralNetwork):
prev = model.seq_length
training_dataset_stats = {}
if 'dataset_stats' in model.stats:
training_dataset_stats = model.stats['dataset_stats']
X_predict = DatasetsApi.get_single_instance_dataset(
body, stats=training_dataset_stats, prev=prev)
if X_predict is None:
return False, []
feature_names = set(training_dataset_stats.keys())
dataset_features = set(X_predict)
dataset_features.discard('DateTime')
if feature_names != dataset_features:
print(feature_names)
print(dataset_features)
return False, []
else:
return False, []
predictions = model.predict(X_predict, uncertainty=True)
MainTransformer.unnormalize(X_predict,
training_dataset_stats,
inplace=True)
MainTransformer.remove_periodic_f(X_predict)
X_predict.loc[:, 'Pollutant'] = Series([x[0] for x in predictions],
index=X_predict.index)
X_predict.loc[:,
'Uncertainty'] = Series([x[1] for x in predictions],
index=X_predict.index)
# add predictions to the DB
keys_with_data_to_be_added = {}
if 'data' in body:
optional_data_keyset = set(body['data'].keys())
dataframe_optional_data = set(X_predict.keys()).difference(
ModelApi.REQUIRED_FIELDS)
keys_with_data_to_be_added = optional_data_keyset.intersection(
dataframe_optional_data)
results = []
for index, row in X_predict.iterrows():
if row['Pollutant'] is not None and math.isnan(
row['Pollutant']):
continue
input_instance = {
'date_time': index,
'longitude': row['Longitude'],
'latitude': row['Latitude'],
'pollutant': body['pollutant'],
'pollution_value': row['Pollutant'],
'uncertainty': row['Uncertainty'],
'data': {}
}
if 'data' in body:
for key in keys_with_data_to_be_added:
input_instance['data'][key] = row[key]
result = DatasetsApi.insert_single_instance(input_instance,
predicted=True)
result = DatasetsApi.insert_single_prediction(input_instance)
results.append(result)
predictions = ModelApi.__predictions_to_primitive_float(
predictions)
print('failed following: ')
print(list(filter(lambda x: not x[0], results)))
return True, predictions
return False, predictions # in case that model does not exist