def calculate_average_daily_measurements_for_last_28_days():
ms = monitoring_site.MonitoringSite()
gr = graph.Graph()
monitoring_sites = list(
app.mongo.db.monitoring_site.find({}, {
"DeviceCode": 1,
"Parish": 1,
"LocationCode": 1,
"Division": 1,
"_id": 0
}))
devices_historical_records = []
for monitoring_site_device in monitoring_sites:
print(monitoring_site_device)
code = monitoring_site_device['DeviceCode']
historical_results = []
records = []
pm25_daily_values = []
average_pm25 = 0
if code: #check if code is not empty
print(code)
parish = monitoring_site_device['Parish']
division = monitoring_site_device['Division']
location_code = monitoring_site_device['LocationCode']
created_at = helpers.str_to_date(
helpers.date_to_str(datetime.now()))
endtime = helpers.date_to_str(datetime.now())
starttime = helpers.date_to_str(datetime.now() -
timedelta(days=28))
monitoring_site_measurements_cursor = gr.get_filtered_data(
code, starttime, endtime, 'daily', 'PM 2.5')
#monitoring_site_measurements_cursor = ms.get_device_past_28_days_measurements(code)
for site in monitoring_site_measurements_cursor:
record = {
'pm2_5_value': int(site['pollutant_value']),
'time': site["time"]
}
records.append(record)
pm25_daily_values.append(int(site['pollutant_value']))
historical_results.append(site)
if len(pm25_daily_values) > 0:
average_pm25 = np.mean(pm25_daily_values)
historical_record = {
'deviceCode': code,
'average_pm25': average_pm25,
'historical_records': records,
'Parish': parish,
'Division': division,
'LocationCode': location_code,
'created_at': created_at
}
devices_historical_records.append(historical_record)
mongo_helpers.save_device_daily_historical_averages(
devices_historical_records)
return jsonify({'response': 'all new hourly measurements saved'}), 200
def get_all_devices_past_28_days_exceedences(self,
pollutant='PM 2.5',
standard='AQI'):
"""
Gets all the exceedences for all the locations in the past 28 days from current day.
Args:
pollutant: the pollutant whose exceedences are to be returned.
standard: the standard to use to get the exceedences.
Returns:
A list of the number of daily exceedences for the specified pollutant and standard in the past 28 days.
"""
created_at = helpers.str_to_date(
helpers.date_to_str(datetime.now().date()))
# print(created_at)
query = {
'$match': {
'created_at': {
'$gte': created_at
},
'pollutant': pollutant,
'standard': standard
}
}
projection = {'$project': {'_id': 0}}
results = list(
app.mongo.db.device_daily_exceedences.aggregate(
[query, projection]))
return results
def get_device_codes():
devices_codes = list(mongo.db.devices.find({}, {"code": 1, "_id": 0}))
devices_codes_list = []
for device_code in devices_codes[0:2]:
last_time = mongo_helpers.get_last_time_from_device_hourly_measurements(
device_code['code'])
start_time = helpers.date_to_str(
helpers.str_to_date(last_time) + timedelta(hours=1))
devices_codes_list.append({
"code": device_code['code'],
"start time": start_time,
"last time": last_time
})
return jsonify({'device codes': devices_codes_list}), 200
def get_all_devices_past_28_days_measurements(self):
"""
Gets all the daily measurements for the deviceo for the past 28 days from current day.
Args:
device_code: the code of the devices whose measurements are to be returned.
Returns:
A list of the daily measurements for the past 28 days.
"""
created_at = helpers.str_to_date(helpers.date_to_str(datetime.now().date()))
print(created_at)
query = {'$match':{ 'created_at': {'$gte': created_at} }}
projection = { '$project': { '_id': 0 }}
results = list(app.mongo.db.device_daily_historical_averages.aggregate([query, projection]) )
return results
def get_random_location_hourly_customised_chart_data_2():
ms = monitoring_site.MonitoringSite()
gr = graph.Graph()
device_code = 'A743BPWK'
start_date = '2020-04-09T07:00:00.000000Z'
end_date = '2020-05-12T07:00:00.000000Z'
frequency = 'monthly'
pollutant = 'PM 2.5'
chart_type = 'pie'
organisation_name = 'KCCA'
parish = ' Wandegeya'
location_code = 'KCCA_KWPE_AQ05'
division = 'Kawempe'
custom_chat_data = []
datasets = []
colors = ['#7F7F7F', '#E377C2', '#17BECF',
'#BCBD22', '#3f51b5'] # blue,cyan, olive,
custom_chart_title = 'Mean ' + frequency.capitalize() + ' ' + \
pollutant + ' for '
locations_names = parish
custom_chart_title = custom_chart_title + locations_names + ' Between ' + helpers.convert_date_to_formated_str(helpers.str_to_date(
start_date), frequency) + ' and ' + helpers.convert_date_to_formated_str(helpers.str_to_date(end_date), frequency)
values = []
labels = []
device_results = {}
filtered_data = gr.get_filtered_data(
device_code, start_date, end_date, frequency, pollutant)
if filtered_data:
for data in filtered_data:
values.append(data['pollutant_value'])
labels.append(data['time'])
device_results = {'pollutant_values': values, 'labels': labels}
color = colors.pop()
dataset = {'data': values, 'label': parish + ' ' + pollutant,
'borderColor': color, 'backgroundColor': color, 'fill': False}
datasets.append(dataset)
custom_chat_data.append({'start_date': start_date, 'end_date': end_date, 'division': division,
'parish': parish, 'frequency': frequency, 'pollutant': pollutant,
'location_code': location_code, 'chart_type': chart_type, 'chart_data': device_results})
return jsonify({'results': custom_chat_data, 'datasets': datasets, 'custom_chart_title': custom_chart_title})
def get_filtered_data(self,
device_code,
start_date=None,
end_date=None,
frequency='daily',
pollutant='PM 2.5'):
"""
Gets all the data for the specified pollutant from the device with the specified code observed between
the specified start date and end date for the specified time frequency.
Args:
device_code (str): the code used to identify a device.
start_date (datetime): the datetime from which observations to be returned should start(lower boundary).
end_date (datetime): the datetime from which observations to be returned should end(upper boundary).
frequency (str): the frequency of the observataions i.e. hourly, daily, monthly.
pollutant (str): the pollutant whose observatations are to be returned i.e. PM 2.5, PM 10, NO2.
Returns:
A list of the data(pollutant values & their corresponding time) for the specified pollutant from the device with the specified code observed between
the specified start date and end date for the specified time frequency.
"""
if start_date == None:
start = helpers.str_to_date_find('2019-06-01T00:00:00Z')
else:
start = helpers.str_to_date(start_date)
if end_date == None:
end = datetime.now()
else:
end = helpers.str_to_date(end_date)
query = {
'$match': {
'deviceCode': device_code,
'time': {
'$lte': end,
'$gte': start
}
}
}
sort_order = {'$sort': {'time': 1}}
time_format = '%Y-%m-%dT%H:%M:%S%z'
if frequency == 'daily':
time_format = '%Y-%m-%d'
elif frequency == 'hourly':
time_format = '%Y-%m-%d %H:%M'
if pollutant == 'PM 10':
projection = {
'$project': {
'_id': 0,
'time': {
'$dateToString': {
'format': time_format,
'date': '$time',
'timezone': 'Africa/Kampala'
}
},
'pollutant_value': {
'$round': ['$characteristics.pm10ConcMass.value', 2]
}
}
}
elif pollutant == 'NO2':
projection = {
'$project': {
'_id': 0,
'time': {
'$dateToString': {
'format': time_format,
'date': '$time',
'timezone': 'Africa/Kampala'
}
},
'pollutant_value': {
'$round': ['$characteristics.no2Conc.value', 2]
}
}
}
else:
projection = {
'$project': {
'_id': 0,
'time': {
'$dateToString': {
'format': time_format,
'date': '$time',
'timezone': 'Africa/Kampala'
}
},
'pollutant_value': {
'$round': ['$characteristics.pm2_5ConcMass.value', 2]
}
}
}
if frequency == 'hourly':
records = app.mongo.db.device_hourly_measurements.aggregate(
[query, projection, sort_order])
elif frequency == 'monthly':
results = list(
app.mongo.db.device_daily_measurements.aggregate(
[query, projection, sort_order]))
records = self.resample_timeseries_data(results, 'M', 'time', 2)
else:
records = app.mongo.db.device_daily_measurements.aggregate(
[query, projection, sort_order])
return list(records)
def generate_customised_chart_data():
ms = monitoring_site.MonitoringSite()
gr = graph.Graph()
if request.method == 'POST':
json_data = request.get_json()
if not json_data:
return {'message': 'No input data provided'}, 400
# input_data, errors = validate_inputs(input_data=json_data) //add server side validation
# if not errors:
locations = json_data["locations"]
start_date = json_data["startDate"]
end_date = json_data["endDate"]
frequency = json_data["frequency"]
pollutant = json_data["pollutant"]
chart_type = json_data["chartType"]
organisation_name = json_data["organisation_name"]
custom_chat_data = []
datasets = [] #displaying multiple locations
locations_devices =[]
colors =['#7F7F7F','#E377C2', '#17BECF', '#BCBD22','#3f51b5']
custom_chart_title= 'Mean ' + frequency.capitalize() + ' '+ pollutant + ' for '
locations_names = ','.join([str(location['label']) for location in locations])
custom_chart_title = custom_chart_title + locations_names
custom_chart_title_second_section = ' Between ' + helpers.convert_date_to_formated_str(helpers.str_to_date(start_date),frequency) + ' and ' + helpers.convert_date_to_formated_str(helpers.str_to_date(end_date),frequency)
for location in locations:
devices = ms.get_location_devices_code( organisation_name, location['label'])
for device in devices:
device_code = device['DeviceCode']
division = device['Division']
parish = device['Parish']
location_code= device['LocationCode']
values =[]
labels =[]
background_colors= []
device_results={}
if chart_type == 'pie':
filtered_data = gr.get_piechart_data(
device_code, start_date, end_date, frequency, pollutant)
if filtered_data:
for data in filtered_data:
values.append(data['category_count'])
labels.append(data['category_name'])
background_colors.append(
helpers.assign_color_to_pollutant_category(data['category_name']))
device_results = {
'pollutant_values': values, 'labels': labels}
color = colors.pop()
dataset = {'data': values, 'label': parish + ' ' +
pollutant, 'backgroundColor': background_colors}
datasets.append(dataset)
custom_chat_data.append({'start_date':start_date, 'end_date':end_date, 'division':division,
'parish':parish,'frequency':frequency, 'pollutant':pollutant,
'location_code':location_code, 'chart_type':chart_type,'chart_data':device_results, 'datasets':datasets, 'custom_chart_title':custom_chart_title, 'custom_chart_title_second_section':custom_chart_title_second_section})
else:
filtered_data = gr.get_filtered_data(device_code, start_date, end_date, frequency, pollutant)
if filtered_data:
for data in filtered_data:
values.append(data['pollutant_value'])
labels.append(data['time'])
device_results = {
'pollutant_values': values, 'labels': labels}
color = colors.pop()
dataset = {'data': values, 'label': parish + ' ' + pollutant,
'borderColor': color, 'backgroundColor': color, 'fill': False}
datasets.append(dataset)
measurement_units = '(µg/m3)'
if pollutant == 'NO2':
measurement_units = ' Concentration'
chart_label = pollutant + measurement_units
custom_chat_data.append({'start_date':start_date, 'end_date':end_date, 'division':division,
'parish':parish,'frequency':frequency, 'pollutant':pollutant,
'location_code':location_code, 'chart_type':chart_type,'chart_data':device_results,
'datasets':datasets, 'custom_chart_title':custom_chart_title, 'chart_label':chart_label, 'custom_chart_title_second_section':custom_chart_title_second_section})
locations_devices.append(devices)
return jsonify({'results':custom_chat_data, 'datasets':datasets, 'custom_chart_title':custom_chart_title, 'custom_chart_title_second_section':custom_chart_title_second_section})
def get_filtered_data(device_code,
start_date=None,
end_date=None,
frequency='daily',
pollutant='PM 2.5'):
"""
returns the data of a certain device with specified parameters
"""
db = connect_mongo()
if start_date == None:
start = helpers.str_to_date_find('2019-06-01T00:00:00Z')
else:
start = helpers.str_to_date(start_date)
if end_date == None:
end = datetime.now()
else:
end = helpers.str_to_date(end_date)
query = {'deviceCode': device_code, 'time': {'$lte': end, '$gte': start}}
if pollutant == 'PM 10':
projection = {
'_id': 0,
'time': 1,
'characteristics.pm10ConcMass.value': 1
}
if frequency == 'hourly':
records = list(
db.device_hourly_measurements.find(query, projection))
elif frequency == 'daily':
records = list(db.device_daily_measurements.find(
query, projection))
else:
records = list(db.device_raw_measurements.find(query, projection))
for i in range(len(records)):
if records[i]['characteristics']['pm10ConcMass'][
'value'] > 0 and records[i]['characteristics'][
'pm10ConcMass']['value'] <= 54:
records[i]['backgroundColor'] = 'green'
elif records[i]['characteristics']['pm10ConcMass'][
'value'] > 54 and records[i]['characteristics'][
'pm10ConcMass']['value'] <= 154:
records[i]['backgroundColor'] = 'yellow'
elif records[i]['characteristics']['pm10ConcMass'][
'value'] > 154 and records[i]['characteristics'][
'pm10ConcMass']['value'] <= 254:
records[i]['backgroundColor'] = 'orange'
elif records[i]['characteristics']['pm10ConcMass'][
'value'] > 254 and records[i]['characteristics'][
'pm10ConcMass']['value'] <= 354:
records[i]['backgroundColor'] = 'red'
elif records[i]['characteristics']['pm10ConcMass'][
'value'] > 354 and records[i]['characteristics'][
'pm10ConcMass']['value'] <= 424:
records[i]['backgroundColor'] = 'purple'
elif records[i]['characteristics']['pm10ConcMass'][
'value'] > 424 and records[i]['characteristics'][
'pm10ConcMass']['value'] <= 604:
records[i]['backgroundColor'] = 'maroon'
else:
records[i]['backgroundColor'] = 'gray'
return records
elif pollutant == 'NO2':
projection = {'_id': 0, 'time': 1, 'characteristics.no2Conc.value': 1}
if frequency == 'hourly':
records = list(
db.device_hourly_measurements.find(query, projection))
elif frequency == 'daily':
records = list(db.device_daily_measurements.find(
query, projection))
else:
records = list(db.device_raw_measurements.find(query, projection))
for i in range(len(records)):
if records[i]['characteristics']['no2Conc'][
'value'] > 0.0 and records[i]['characteristics'][
'no2Conc']['value'] <= 53:
records[i]['backgroundColor'] = 'green'
elif records[i]['characteristics']['no2Conc'][
'value'] > 53 and records[i]['characteristics']['no2Conc'][
'value'] <= 100:
records[i]['backgroundColor'] = 'yellow'
elif records[i]['characteristics']['no2Conc'][
'value'] > 100 and records[i]['characteristics'][
'no2Conc']['value'] <= 360:
records[i]['backgroundColor'] = 'orange'
elif records[i]['characteristics']['no2Conc'][
'value'] > 360 and records[i]['characteristics'][
'no2Conc']['value'] <= 649:
records[i]['backgroundColor'] = 'red'
elif records[i]['characteristics']['no2Conc'][
'value'] > 649 and records[i]['characteristics'][
'no2Conc']['value'] <= 1249:
records[i]['backgroundColor'] = 'purple'
elif records[i]['characteristics']['no2Conc'][
'value'] > 1249 and records[i]['characteristics'][
'no2Conc']['value'] <= 2049:
records[i]['backgroundColor'] = 'maroon'
else:
records[i]['backgroundColor'] = 'gray'
return records
else:
projection = {
'_id': 0,
'time': 1,
'characteristics.pm2_5ConcMass.value': 1
}
if frequency == 'hourly':
records = list(
db.device_hourly_measurements.find(query, projection))
elif frequency == 'daily':
records = list(db.device_daily_measurements.find(
query, projection))
else:
records = list(db.device_raw_measurements.find(query, projection))
for i in range(len(records)):
if records[i]['characteristics']['pm2_5ConcMass'][
'value'] > 0.0 and records[i]['characteristics'][
'pm2_5ConcMass']['value'] <= 12.0:
records[i]['backgroundColor'] = 'green'
elif records[i]['characteristics']['pm2_5ConcMass'][
'value'] > 12.0 and records[i]['characteristics'][
'pm2_5ConcMass']['value'] <= 35.4:
records[i]['backgroundColor'] = 'yellow'
elif records[i]['characteristics']['pm2_5ConcMass'][
'value'] > 35.4 and records[i]['characteristics'][
'pm2_5ConcMass']['value'] <= 55.4:
records[i]['backgroundColor'] = 'orange'
elif records[i]['characteristics']['pm2_5ConcMass'][
'value'] > 55.4 and records[i]['characteristics'][
'pm2_5ConcMass']['value'] <= 150.4:
records[i]['backgroundColor'] = 'red'
elif records[i]['characteristics']['pm2_5ConcMass'][
'value'] > 150.4 and records[i]['characteristics'][
'pm2_5ConcMass']['value'] <= 250.4:
records[i]['backgroundColor'] = 'purple'
elif records[i]['characteristics']['pm2_5ConcMass'][
'value'] > 250.4 and records[i]['characteristics'][
'pm2_5ConcMass']['value'] <= 500.4:
records[i]['backgroundColor'] = 'maroon'
else:
records[i]['backgroundColor'] = 'gray'
return records