def insert_dataset(files):
"""
Function for inserting a whole dataset in the database
:param files: dict with FileStorage instances, holding datasets' files
:return: (True, None) | (False, str) - string instance is the error message
"""
# parameters required for basic data such as which dataset to be improted, what time formats to be used, etc.
BASE_PARAMS = ['Date', 'Time']
# parameters required for getting specific columns from given datasets, etc. for Temperature get tempC column
DATASET_PARAMS = ['weatherFormat', 'pollutantFormat']
dataset_metadata = json.load(files['metadata'])
if not isinstance(dataset_metadata, dict):
return False, Errors.WRONG_INSTANCE.value
are_params_missing = Helpers.are_params_missing(
dataset_metadata, BASE_PARAMS + DATASET_PARAMS)
if are_params_missing:
return False, Errors.MISSING_PARAM.value
for x in DATASET_PARAMS:
if not isinstance(dataset_metadata[x], dict):
return False, Errors.WRONG_INSTANCE.value
for key in files:
dataset_metadata[key + 'Datasets'] = files[key]
# Combine multiple datasets and get result
main_transformer = MainTransformer(config=dataset_metadata)
main_transformer.add_transformer(Transformers.WEATHER_TRANSFORMER)
main_transformer.add_transformer(Transformers.POLLUTANT_TRANSFORMER)
main_transformer.transform()
dataset = main_transformer.get_dataset()
result, err = DBManager.insert_dataset(dataset, dataset_metadata)
return result, err
import pandas
import random
import numpy as np
import json
from airpyllution import MainTransformer
from airpyllution import Transformers
from airpyllution import DBManager
with open('configTwo.json') as file:
dataset_one = json.load(file)
with open('configOne.json') as file:
dataset_two = json.load(file)
data_transformer = MainTransformer(config=dataset_one)
data_transformer.add_transformer(Transformers.WEATHER_TRANSFORMER)
data_transformer.add_transformer(Transformers.POLLUTANT_TRANSFORMER)
data_transformer.transform()
dataset_centre = data_transformer.get_dataset()
data_transformer = MainTransformer(config=dataset_two)
data_transformer.add_transformer(Transformers.WEATHER_TRANSFORMER)
data_transformer.add_transformer(Transformers.POLLUTANT_TRANSFORMER)
data_transformer.transform()
dataset_a33 = data_transformer.get_dataset()
length_centre = dataset_centre.shape[0]
length_a33 = dataset_a33.shape[0]
dataset_centre['Longitude'] = -1.463484