def make_forecasts_for_Sander():
"""2018 August: Hei igjen Ragnar.
Har du statistikk på varsla faregrad over ein heil sesong for Noreg? Eit snitt. XX dagar med faregrad 1,
XX dagar med faregrad 2, XX dagar med faregrad 3.... fordelt på XX varslingsdagar.
:return:
"""
pickle_file_name = '{0}201808_avalanche_forecasts_sander.pickle'.format(
env.local_storage)
get_new = False
all_dangers = []
if get_new:
years = ['2012-13', '2013-14', '2014-15', '2015-16']
for y in years:
from_date, to_date = gm.get_forecast_dates(y)
region_ids = gm.get_forecast_regions(y, get_b_regions=True)
all_dangers += gd.get_forecasted_dangers(region_ids, from_date,
to_date)
years = ['2016-17', '2017-18']
for y in years:
from_date, to_date = gm.get_forecast_dates(y)
region_ids = gm.get_forecast_regions(y, get_b_regions=True)
all_dangers += gd.get_forecasted_dangers(region_ids, from_date,
to_date)
mp.pickle_anything(all_dangers, pickle_file_name)
else:
all_dangers = mp.unpickle_anything(pickle_file_name)
output_forecast_problems = '{0}201808 Faregrader for Sander.txt'.format(
env.output_folder)
import pandas as pd
all_dangers_dict = []
for a in all_dangers:
all_dangers_dict.append(a.__dict__)
col_names = list(all_dangers_dict[0].keys())
all_dangers_df = pd.DataFrame(all_dangers_dict,
columns=col_names,
index=range(0, len(all_dangers_dict), 1))
a = 1
def make_all_2015_16():
## Get new or load from pickle.
get_new = True
## Use already made data set. Remember to make get_new = False
make_new_incident_list = True
make_new_node_list = True
## Set dates
from_date = dt.date(2015, 11, 30)
to_date = dt.date(2016, 6, 1)
# to_date = dt.date.today()
## Get regions
# region_id = [112, 117, 116, 128]
region_ids = gm.get_forecast_regions(year='2015-16')
## The output
incident_ap_dl_json = '{0}incident_and_forecast.json'.format(
env.output_folder)
incident_ap_dl_csv = '{0}incident_and_forecast.csv'.format(
env.output_folder)
make_dataset(from_date_inn=from_date,
to_date_inn=to_date,
region_ids_inn=region_ids,
incident_ap_dl_csv_inn=incident_ap_dl_csv,
incident_ap_dl_json_inn=incident_ap_dl_json,
get_new_inn=get_new,
make_new_incident_list_inn=make_new_incident_list,
make_new_node_list_inn=make_new_node_list)
def get_2016_17_warnings(how_to_get_data='Get new and dont pickle', pickle_file_name=None):
'''
:param hot_to_get_data: 'Get new and dont pickle', 'Get new and save pickle' or 'Load pickle'
:param file_name: Not needed if no pickles involved
:return:
'''
if 'Get new' in how_to_get_data:
from_date = dt.date(2016, 11, 30)
#to_date = dt.date.today()
to_date = dt.date(2017, 5, 31)
#region_ids = [3012, 3013]
region_ids = gm.get_forecast_regions(year='2016-17')
all_warnings = []
for region_id in region_ids:
all_warnings += gd.get_forecasted_dangers(region_id, from_date, to_date, include_ikke_vurdert=False)
# Sort by date
all_warnings = sorted(all_warnings, key=lambda AvalancheDanger: AvalancheDanger.date)
if 'and save pickle' in how_to_get_data:
mp.pickle_anything(all_warnings, pickle_file_name)
elif 'Load pickle' in how_to_get_data:
all_warnings = mp.unpickle_anything(pickle_warnings_file_name)
else:
all_warnings = 'No valid data retrival method given in get_2015_16_warnings.'
return all_warnings
def make_plots_for_regions(region_ids=None,
year='2019-20',
plot_folder=env.plot_folder + 'regionplots/'):
"""This method prepares data for plotting and calls on the plot methods. Pure administration.
:param region_ids: [int] Forecast region ID as given in ForecastRegionKDV
:param year: [string]
:param plot_folder: [string] Folder for plot files
:return:
"""
if not region_ids:
region_ids = gm.get_forecast_regions(year)
if not isinstance(region_ids, list):
region_ids = [region_ids]
for region_id in region_ids:
problems, dangers, aval_indexes = _get_dl_prob_avindex(region_id,
year=year)
region_name = problems[0].region_name
causes = [] # list of dates and causes
for p in problems:
causes.append(AvalancheCause(p.cause_tid, p.date, p.source))
danger_levels = []
for d in dangers:
if d.nick != 'drift@svv' and d.danger_level > 0: # for these plots elrapp wil make noise.
danger_levels.append(
DangerLevel(d.danger_level, d.date, d.source, d))
# Only data with danger levels are plotted
if len(danger_levels) is not 0:
# Danger level histograms
_plot_danger_levels(region_name,
danger_levels,
aval_indexes,
year=year,
plot_folder=plot_folder)
# Cause horizontal line plots
if int(
year[0:4]
) > 2014: # modern avalanche problems was introduced winter of 2014-15
_plot_causes(region_name,
causes,
year=year,
plot_folder=plot_folder)
def _get_raw_varsom(year, date, days, max_file_age=23):
if date:
season = gm.get_season_from_date(date)
regions = gm.get_forecast_regions(year=season, get_b_regions=True)
aw = []
from_date = date - dt.timedelta(days=days + 1)
to_date = date
single_warning = gf.get_avalanche_warnings(regions, from_date, to_date)
for sw in single_warning:
if sw.danger_level > 0:
aw.append(sw)
else:
aw = gvp.get_all_forecasts(year=year, max_file_age=max_file_age)
return aw
def make_2018_19_plots(plot_folder=env.plot_folder + 'regionplots/'):
"""Makes all plots for all regions and saves to web-app folder."""
from_date = dt.date(2018, 11, 15)
to_date = dt.date.today() + dt.timedelta(days=2)
if dt.date.today().year == 2018:
to_date = dt.date(2019, 1, 1)
region_ids = gm.get_forecast_regions('2018-19')
for i in region_ids:
problems, dangers, aval_indexes = get_data(i,
from_date,
to_date,
get_new=True)
make_plots_for_region(i,
problems,
dangers,
aval_indexes,
from_date,
to_date,
plot_folder=plot_folder)
def _get_all_snow(get_new=False):
file_name = '{}observations and forecasts 2012-17.pickle'.format(
env.local_storage)
if get_new:
all_observations = go.get_all_registrations('2012-12-01',
'2017-07-01',
geohazard_tids=10)
years = ['2012-13', '2013-14', '2014-15', '2015-16', '2016-17']
all_forecasts = []
for y in years:
from_date, to_date = gm.get_forecast_dates(y)
region_ids = gm.get_forecast_regions(y)
all_forecasts += gfa.get_avalanche_warnings(
region_ids, from_date, to_date)
mp.pickle_anything([all_observations, all_forecasts], file_name)
else:
[all_observations, all_forecasts] = mp.unpickle_anything(file_name)
return all_observations, all_forecasts
def get_all_forecasts(year, lang_key=1, max_file_age=23):
"""Specialized method for getting all forecasts for one season.
For the current season (at the time of writing, 2018-19), if a request
has been made the last 23hrs, data is retrieved from a locally stored pickle,
if not, new request is made to the regObs api. Previous seasons are not
requested if a pickle is found in local storage.
:param year: [string] Eg. season '2017-18'
:param lang_key [int] 1 is norwegian, 2 is english
:param max_file_age: [int] hrs how old the file is before new is retrieved
:return valid_forecasts: [list of AvalancheWarning]
"""
from_date, to_date = gm.get_forecast_dates(year=year)
file_name = '{0}all_forecasts_{1}_lk{2}.pickle'.format(
env.local_storage, year, lang_key)
file_date_limit = dt.datetime.now() - dt.timedelta(hours=max_file_age)
# if we are well out of the current season (30 days) its little chance the data set has changed.
current_season = gm.get_season_from_date(dt.date.today() -
dt.timedelta(30))
# Get forecast regions used in the current year
region_ids = gm.get_forecast_regions(year, get_b_regions=True)
get_new = True
if os.path.exists(file_name):
# if file contains a season long gone, dont make new.
if year == current_season:
file_age = dt.datetime.fromtimestamp(os.path.getmtime(file_name))
# If file is newer than the given time limit, dont make new.
if file_age > file_date_limit:
# If file size larger than that of an nearly empty file, dont make new.
if os.path.getsize(file_name) > 100: # 100 bytes limit
get_new = False
else:
get_new = False
if get_new:
lg.info(
"getvarsompickles.py -> get_all_forecasts: Get new {0} forecasts and pickle."
.format(year))
all_forecasts = gfa.get_avalanche_warnings(region_ids,
from_date,
to_date,
lang_key=lang_key)
# Valid forecasts have a danger level. The other are empty.
valid_forecasts = []
for f in all_forecasts:
if f.danger_level > 0:
valid_forecasts.append(f)
mp.pickle_anything(valid_forecasts, file_name)
else:
valid_forecasts = mp.unpickle_anything(file_name)
return valid_forecasts
def make_avalanche_problemes_for_techel():
"""Gets forecastes and observed avalanche problems and dangers for Frank Techel.
Takes 20-30 min to run a year.
:return:
"""
pickle_file_name = '{0}runavalancheproblems_techel.pickle'.format(
env.local_storage)
years = ['2014-15', '2015-16', '2016-17', '2017-18']
get_new = False
if get_new:
forecast_problems = []
forecast_dangers = []
observed_dangers = []
observed_problems = []
for y in years:
# Get forecast data. Different region ids from year to year.
region_ids = gm.get_forecast_regions(year=y)
from_date, to_date = gm.get_forecast_dates(y)
forecast_problems += gp.get_forecasted_problems(region_ids,
from_date,
to_date,
lang_key=2)
forecast_dangers += gd.get_forecasted_dangers(region_ids,
from_date,
to_date,
lang_key=2)
# Get observed data. All older data in regObs have been mapped to new regions.
region_ids = gm.get_forecast_regions(year='2016-17')
from_date, to_date = gm.get_forecast_dates(
y, padding=dt.timedelta(days=20))
this_years_observed_dangers = gd.get_observed_dangers(region_ids,
from_date,
to_date,
lang_key=2)
this_years_observed_problems = gp.get_observed_problems(region_ids,
from_date,
to_date,
lang_key=2)
# Update observations with forecast region ids and names used the respective years
for od in this_years_observed_dangers:
utm33x = od.metadata['Original data'].UTMEast
utm33y = od.metadata['Original data'].UTMNorth
region_id, region_name = gm.get_forecast_region_for_coordinate(
utm33x, utm33y, y)
od.region_regobs_id = region_id
od.region_name = region_name
for op in this_years_observed_problems:
utm33x = op.metadata['Original data']['UtmEast']
utm33y = op.metadata['Original data']['UtmNorth']
region_id, region_name = gm.get_forecast_region_for_coordinate(
utm33x, utm33y, y)
op.region_regobs_id = region_id
op.region_name = region_name
observed_dangers += this_years_observed_dangers
observed_problems += this_years_observed_problems
mp.pickle_anything([
forecast_problems, forecast_dangers, observed_dangers,
observed_problems
], pickle_file_name)
else:
[
forecast_problems, forecast_dangers, observed_dangers,
observed_problems
] = mp.unpickle_anything(pickle_file_name)
# Run EAWS mapping on all problems
for p in forecast_problems:
p.map_to_eaws_problems()
for p in observed_problems:
p.map_to_eaws_problems()
output_forecast_problems = '{0}Techel forecast problems.csv'.format(
env.output_folder)
output_forecast_dangers = '{0}Techel forecast dangers.csv'.format(
env.output_folder)
output_observed_problems = '{0}Techel observed problems.csv'.format(
env.output_folder)
output_observed_dangers = '{0}Techel observed dangers.csv'.format(
env.output_folder)
import collections as coll
# Write observed dangers to file
with open(output_observed_dangers, 'w', encoding='utf-8') as f:
make_header = True
for d in observed_dangers:
out_data = coll.OrderedDict([
('Date', dt.date.strftime(d.date, '%Y-%m-%d')),
('Reg time',
dt.datetime.strftime(d.registration_time, '%Y-%m-%d %H:%M')),
('Region id', d.region_regobs_id),
('Region', d.region_name),
('Municipal', d.municipal_name),
('Nick', d.nick),
('Competence', d.competence_level),
('DL', d.danger_level),
('Danger level', d.danger_level_name),
('Forecast correct', d.forecast_correct),
# ('Table', d.data_table),
# ('URL', d.url),
])
if make_header:
f.write(' ;'.join([fe.make_str(d)
for d in out_data.keys()]) + '\n')
make_header = False
f.write(' ;'.join([fe.make_str(d)
for d in out_data.values()]) + '\n')
# Write forecasted dangers to file
with open(output_forecast_dangers, 'w', encoding='utf-8') as f:
make_header = True
for d in forecast_dangers:
out_data = coll.OrderedDict([
('Date', dt.date.strftime(d.date, '%Y-%m-%d')),
('Region id', d.region_regobs_id),
('Region', d.region_name),
('Nick', d.nick),
('DL', d.danger_level),
('Danger level', d.danger_level_name),
# ('Table', d.data_table),
# ('URL', d.url),
('Main message', ' '.join(d.main_message_en.split()))
])
if make_header:
f.write(' ;'.join([fe.make_str(d)
for d in out_data.keys()]) + '\n')
make_header = False
f.write(' ;'.join([fe.make_str(d)
for d in out_data.values()]) + '\n')
# Write forecasted problems to file
with open(output_forecast_problems, 'w', encoding='utf-8') as f:
make_header = True
for p in forecast_problems:
out_data = coll.OrderedDict([
('Date', dt.date.strftime(p.date, '%Y-%m-%d')),
('Region id', p.region_regobs_id),
('Region', p.region_name),
('Nick', p.nick_name),
('Problem order', p.order),
('Problem', p.problem),
('EAWS problem', p.eaws_problem),
('Cause/ weaklayer', p.cause_name),
# ('TypeTID', p.aval_type_tid),
('Type', p.aval_type),
('Size', p.aval_size),
('Trigger', p.aval_trigger),
('Probability', p.aval_probability),
('Distribution', p.aval_distribution),
('DL', p.danger_level),
('Danger level', p.danger_level_name),
# ('Table', p.regobs_table),
# ('URL', p.url)
])
if make_header:
f.write(' ;'.join([fe.make_str(d)
for d in out_data.keys()]) + '\n')
make_header = False
f.write(' ;'.join([fe.make_str(d)
for d in out_data.values()]) + '\n')
# Write observed problems to file
with open(output_observed_problems, 'w', encoding='utf-8') as f:
make_header = True
for p in observed_problems:
out_data = coll.OrderedDict([
('Date', dt.date.strftime(p.date, '%Y-%m-%d')),
('Reg time',
dt.datetime.strftime(p.registration_time, '%Y-%m-%d %H:%M')),
('Region id', p.region_regobs_id),
('Region', p.region_name),
('Municipal', p.municipal_name),
('Nick', p.nick_name),
('Competence', p.competence_level),
# ('Problem order', p.order),
('EAWS problem', p.eaws_problem),
('Cause/ weaklayer', p.cause_name),
# ('TypeTID', p.aval_type_tid),
('Type', p.aval_type),
('Catch 1', p.cause_attribute_crystal),
('Catch 2', p.cause_attribute_light),
('Catch 3', p.cause_attribute_soft),
('Catch 4', p.cause_attribute_thin),
('Size', p.aval_size),
('Trigger', p.aval_trigger),
# ('Probability', p.aval_probability),
# ('Distribution', p.aval_distribution),
# ('RegID', p.regid),
# ('Table', p.regobs_table),
# ('URL', p.url)
])
if make_header:
f.write(' ;'.join([fe.make_str(d)
for d in out_data.keys()]) + '\n')
make_header = False
f.write(' ;'.join([fe.make_str(d)
for d in out_data.values()]) + '\n')
for (key, val) in aval_types)
aval_triggers = ', '.join("{!s} ({!r})".format(key, val)
for (key, val) in aval_triggers)
s = u'{0};{1};{2};{3};{4};{5};{6};{7}\n'.format(
m.occurrences, danger_levels, problems, aval_triggers, cause_names,
aval_types, m.main_message_no, m.main_message_en)
l.write(s)
l.close()
if __name__ == "__main__":
year = '2017-18'
regions = gm.get_forecast_regions(year=year)
date_from, date_to = gm.get_dates_from_season(year=year)
# file names
file_name_for_warnings_pickle = '{0}{1}'.format(
env.local_storage, 'runmainmessage warnings.pickle')
file_name_for_main_messages_pickle = '{0}{1}'.format(
env.local_storage, 'runmainmessage main messages.pickle')
file_name_for_main_messages_csv = '{0}{1}'.format(
env.output_folder, 'Alle hovedbudskap {}.csv'.format(year))
##### pickle the warnings and dataset with main messages
# pickle_warnings(regions, date_from, date_to, file_name_for_warnings_pickle)
main_messages = select_messages_with_more(file_name_for_warnings_pickle)
mp.pickle_anything(main_messages, file_name_for_main_messages_pickle)
main_messages = mp.unpickle_anything(file_name_for_main_messages_pickle)
def make_dl_incident_markus():
"""
From the beginning of time:
get all forecasts.
and then get how many on dl 3.
get all incidents,
excpt elrapp, and all in back country
all these, get all on days in regions of dl 3.
get all with serious caracter on days and in regions on dl 3
:return:
"""
pickle_file_name = '{0}incident_on_dl3_for_markus.pickle'.format(
env.local_storage)
years = ['2012-13', '2013-14', '2014-15', '2015-16', '2016-17']
get_new = False
all_dangers = []
all_incidents = []
if get_new:
for y in years:
# get forecast regions used this year
from_date, to_date = gm.get_forecast_dates(y)
# get incidents for this year and map to this years forecast regions
this_year_incidents = go.get_incident(from_date,
to_date,
geohazard_tids=10)
for i in this_year_incidents:
utm33x = i.UTMEast
utm33y = i.UTMNorth
region_id, region_name = gm.get_forecast_region_for_coordinate(
utm33x, utm33y, y)
i.region_regobs_id = region_id
i.region_name = region_name
all_incidents += this_year_incidents
# get regions and the forecasts used this year
region_ids = gm.get_forecast_regions(y)
all_dangers += gd.get_forecasted_dangers(region_ids, from_date,
to_date)
# in the end, pickle it all
mp.pickle_anything([all_dangers, all_incidents], pickle_file_name)
else:
[all_dangers, all_incidents] = mp.unpickle_anything(pickle_file_name)
all_dl3 = []
for d in all_dangers:
if d.danger_level == 3:
all_dl3.append(d)
all_back_country_incidents = []
for i in all_incidents:
if 'drift@svv' not in i.NickName:
# if activity influenced is backcounty og scooter
# should probably include 100 which is non specified incidents
# giving this dataset the observations not specified
if i.ActivityInfluencedTID in [
100, 110, 111, 112, 113, 114, 115, 116, 117, 130
]:
all_back_country_incidents.append(i)
all_back_country_incidents_with_consequence = []
for i in all_back_country_incidents:
# If damageextent is nestenulykke, personer skadet eller personer omkommet
if i.DamageExtentTID > 28:
all_back_country_incidents_with_consequence.append(i)
# find incidents in regions on days with danger level 3
# find incidetns in region on day with dl3
all_back_country_incidents_on_region_dl3 = []
all_back_country_incidents_with_consequence_on_region_dl3 = []
for d in all_dl3:
danger_date = d.date
danger_region_id = d.region_regobs_id
for i in all_back_country_incidents:
incident_date = i.DtObsTime.date()
incident_region_id = i.ForecastRegionTID
if incident_date == danger_date and incident_region_id == danger_region_id:
all_back_country_incidents_on_region_dl3.append(i)
for i in all_back_country_incidents_with_consequence:
incident_date = i.DtObsTime.date()
incident_region_id = i.ForecastRegionTID
if incident_date == danger_date and incident_region_id == danger_region_id:
all_back_country_incidents_with_consequence_on_region_dl3.append(
i)
print('Totalt varsler laget siden tidenes morgen: {}'.format(
len(all_dangers)))
print('Totalt varsler på fg 3: {}'.format(len(all_dl3)))
print('Totalt antall hendelser i baklandet: {}'.format(
len(all_back_country_incidents)))
print('Totalt antall hendelser i baklandet med konsekvens: {}'.format(
len(all_back_country_incidents_with_consequence)))
print(
'Totalt antall hendelser i baklandet i regioner på dager med fg3: {}'.
format(len(all_back_country_incidents_on_region_dl3)))
print(
'Totalt antall hendelser i baklandet i regioner på dager med fg3 med konsekvens: {}'
.format(
len(all_back_country_incidents_with_consequence_on_region_dl3)))
return
def __init__(self, regobs_types,
seasons=('2017-18', '2018-19', '2019-20')):
"""
Object contains aggregated data used to generate labeled datasets.
:param regobs_types: Tuple/list of string names for RegObs observation types to fetch.
:param seasons: Tuple/list of string representations of avalanche seasons to fetch.
"""
self.regobs_types = regobs_types
self.tree = {}
aw = []
raw_regobs = {}
for season in seasons:
aw += gvp.get_all_forecasts(year=season)
regions = gm.get_forecast_regions(year=season, get_b_regions=True)
raw_regobs = {
**raw_regobs,
**_get_regobs_obs(regions, season, regobs_types)
}
for forecast in aw:
row = {
# Metadata
'region_id':
forecast.region_id,
'region_name':
forecast.region_name,
'region_type':
forecast.region_type_name,
'date':
forecast.date_valid,
'danger_level':
forecast.danger_level,
'emergency_warning':
float(forecast.emergency_warning == 'Ikke gitt')
}
label = OrderedDict({})
label[('CLASS', _NONE, 'danger_level')] = forecast.danger_level
label[('CLASS', _NONE,
'emergency_warning')] = forecast.emergency_warning
# Weather data
weather = {
'precip_most_exposed':
forecast.mountain_weather.precip_most_exposed,
'precip':
forecast.mountain_weather.precip_region,
'wind_speed':
WIND_SPEEDS.get(forecast.mountain_weather.wind_speed, 0),
'wind_change_speed':
WIND_SPEEDS.get(forecast.mountain_weather.change_wind_speed,
0),
'temp_min':
forecast.mountain_weather.temperature_min,
'temp_max':
forecast.mountain_weather.temperature_max,
'temp_lev':
forecast.mountain_weather.temperature_elevation,
'temp_freeze_lev':
forecast.mountain_weather.freezing_level,
}
# We use multiple loops to get associated values near each other in e.g. .csv-files.
for wind_dir in DIRECTIONS:
weather[f"wind_dir_{wind_dir}"] = float(
forecast.mountain_weather.wind_direction == wind_dir)
for wind_dir in DIRECTIONS:
weather[f"wind_chg_dir_{wind_dir}"] = float(
forecast.mountain_weather.change_wind_direction ==
wind_dir)
hours = [0, 6, 12, 18]
for h in hours:
weather[f"wind_chg_start_{h}"] = float(
forecast.mountain_weather.change_hour_of_day_start == h)
for h in hours:
weather[f"temp_fl_start_{h}"] = float(
forecast.mountain_weather.change_hour_of_day_start == h)
row['weather'] = weather
# Problem data
prb = {}
problem_types = [
PROBLEMS.get(p.avalanche_problem_type_id, _NONE)
for p in forecast.avalanche_problems
]
problems = {}
prb['problem_amount'] = len(forecast.avalanche_problems)
label[('CLASS', _NONE, 'problem_amount')] = prb['problem_amount']
for i in range(1, 4):
label[('CLASS', _NONE, f"problem_{i}")] = _NONE
for problem in PROBLEMS.values():
if problem in problem_types:
index = problem_types.index(problem)
problems[problem] = forecast.avalanche_problems[index]
prb[f"problem_{problem}"] = -(
problems[problem].avalanche_problem_id - 4)
label[('CLASS', _NONE, f"problem_{index + 1}")] = problem
else:
problems[problem] = gf.AvalancheWarningProblem()
prb[f"problem_{problem}"] = 0
for problem in PROBLEMS.values():
p_data = problems[problem]
forecast_cause = CAUSES.get(p_data.aval_cause_id, _NONE)
for cause in CAUSES.values():
prb[f"problem_{problem}_cause_{cause}"] = float(
forecast_cause == cause)
prb[f"problem_{problem}_dsize"] = p_data.destructive_size_ext_id
prb[f"problem_{problem}_prob"] = p_data.aval_probability_id
prb[f"problem_{problem}_trig"] = {
10: 0,
21: 1,
22: 2
}.get(p_data.aval_trigger_simple_id, 0)
prb[f"problem_{problem}_dist"] = p_data.aval_distribution_id
prb[f"problem_{problem}_lev_max"] = p_data.exposed_height_1
prb[f"problem_{problem}_lev_min"] = p_data.exposed_height_2
label[('CLASS', problem, "cause")] = forecast_cause
label[('CLASS', problem,
"dsize")] = p_data.destructive_size_ext_id
label[('CLASS', problem, "prob")] = p_data.aval_probability_id
label[('CLASS', problem,
"trig")] = p_data.aval_trigger_simple_id
label[('CLASS', problem, "dist")] = p_data.aval_distribution_id
label[('CLASS', problem,
"lev_fill")] = p_data.exposed_height_fill
for n in range(1, 5):
prb[f"problem_{problem}_lev_fill{n}"] = float(
p_data.exposed_height_fill == n)
for n in range(0, 8):
aspect_attr_name = f"problem_{problem}_aspect_{DIRECTIONS[n]}"
prb[aspect_attr_name] = float(p_data.valid_expositions[n])
label[('MULTI', problem,
"aspect")] = p_data.valid_expositions.zfill(8)
label[('REAL', problem, "lev_max")] = p_data.exposed_height_1
label[('REAL', problem, "lev_min")] = p_data.exposed_height_2
# Check for consistency
if prb[f"problem_{problem}_lev_min"] > prb[
f"problem_{problem}_lev_max"]:
continue
row['problems'] = prb
row['label'] = label
# RegObs data
row['regobs'] = {}
current_regobs = raw_regobs.get(
(forecast.region_id, forecast.date_valid), {})
# Use 5 most competent observations, and list both categories as well as scalars
for obs_idx in range(0, 5):
# One type of observation (test, danger signs etc.) at a time
for regobs_type in self.regobs_types:
obses = current_regobs[
regobs_type] if regobs_type in current_regobs else []
# Go through each requested class attribute from the specified observation type
for attr, cat in REGOBS_CLASSES[regobs_type].items():
# We handle categories using 1-hot, so we step through each category
for cat_name in cat.values():
attr_name = f"regobs_{REG_ENG[regobs_type]}_{_camel_to_snake(attr)}_{cat_name}_{obs_idx}"
row['regobs'][attr_name] = obses[obs_idx][
cat_name] if len(obses) > obs_idx else 0
# Go through all requested scalars
for attr, (regobs_attr,
conv) in REGOBS_SCALARS[regobs_type].items():
attr_name = f"regobs_{REG_ENG[regobs_type]}_{_camel_to_snake(attr)}_{obs_idx}"
try:
row['regobs'][attr_name] = conv(
obses[obs_idx]
[regobs_attr]) if len(obses) > obs_idx else 0
except TypeError:
row['regobs'][attr_name] = 0
# Check for consistency
if weather['temp_min'] > weather['temp_max']:
continue
self.tree[(forecast.region_id, forecast.date_valid)] = row
def _get_regobs_obs(year,
requested_types,
date=None,
days=None,
max_file_age=23):
regions = gm.get_forecast_regions(year=year, get_b_regions=True)
observations = {}
observations_neigh = {}
varsomdata_obses = {}
if len(requested_types) == 0:
return observations
file_name = f'{se.local_storage}regobs_v{CSV_VERSION}_{year}.pickle'
file_date_limit = dt.datetime.now() - dt.timedelta(hours=max_file_age)
current_season = gm.get_season_from_date(dt.date.today() -
dt.timedelta(30))
number_of_records = 50
get_new = True
try:
# Don't fetch new data if old is cached. If older season file doesn't exists we get out via an exception.
if dt.datetime.fromtimestamp(os.path.getmtime(
file_name)) > file_date_limit or year != current_season:
get_new = False
if date:
get_new = True
except FileNotFoundError:
pass
if date:
from_date = date - dt.timedelta(days=days)
to_date = date
else:
from_date, to_date = gm.get_dates_from_season(year=year)
if "AvalancheIndex" in requested_types:
avalanche_index = True
else:
avalanche_index = False
req_set = set(requested_types) & set(REG_ENG_V4.keys())
# Make sure all requested elements from RegObs actually have the information we need specified
if not min(
map(lambda x: set(list(x.keys())).issuperset(req_set),
[REGOBS_CLASSES, REGOBS_SCALARS, REG_ENG])):
raise RegObsRegTypeError()
url = "https://api.regobs.no/v4/Search"
query = {
"LangKey": 1,
"FromDate": from_date.isoformat(),
"ToDate": to_date.isoformat(),
"SelectedRegistrationTypes": None,
"SelectedRegions": regions,
"NumberOfRecords": number_of_records,
"Offset": 0
}
results = []
def send_req(queries):
query = queries.pop()
try:
req = requests.post(url=url, json=query)
return (req, query)
except:
return (None, query)
if get_new:
future_tuples = []
total_matches = requests.post(url=url + "/Count",
json=query).json()["TotalMatches"]
with futures.ThreadPoolExecutor(140) as executor:
queries = []
while query["Offset"] < total_matches:
queries.append(query.copy())
query["Offset"] += number_of_records
for _ in range(0, len(queries)):
future = executor.submit(send_req, queries)
future_tuples.append((0, future))
while len(future_tuples):
retries, future = future_tuples.pop()
try:
response, query = future.result()
raw_obses = response.json()
except:
if retries < 5:
future = executor.submit(send_req, [query])
future_tuples.insert(0, (retries + 1, future))
else:
offset = json.loads(response.request.body)["Offset"]
print(
f"Failed to fetch regobs, offset {offset}, skipping",
file=sys.stderr)
continue
results = results + raw_obses
if not date:
with open(file_name, 'wb') as handle:
pickle.dump(results, handle, protocol=pickle.HIGHEST_PROTOCOL)
else:
try:
with open(file_name, 'rb') as handle:
results = pickle.load(handle)
except:
os.remove(file_name)
return _get_regobs_obs(regions, year, requested_types,
max_file_age)
for raw_obs in results:
date = dt.datetime.fromisoformat(raw_obs["DtObsTime"]).date()
key = (date.isoformat(), raw_obs["ObsLocation"]["ForecastRegionTID"])
if key not in observations:
observations[key] = {}
observations_neigh[key] = {}
varsomdata_obses[key] = []
for obs_type in req_set:
if REG_ENG_V4[obs_type] not in raw_obs or not raw_obs[
REG_ENG_V4[obs_type]]:
continue
reg = raw_obs[REG_ENG_V4[obs_type]]
# Ignore snow profiles of the old format
if obs_type == "Snøprofil" and "StratProfile" not in reg:
continue
obs = {"competence": raw_obs["Observer"]["CompetenceLevelTID"]}
try:
for attr, categories in REGOBS_CLASSES[obs_type].items():
for cat_id, cat_name in categories.items():
if isinstance(reg, list):
obs[cat_name] = 1 if cat_id in map(
lambda x: x[attr], reg) else 0
else:
obs[cat_name] = 1 if cat_id == reg[attr] else 0
except KeyError:
pass
try:
for regobs_attr, conv in REGOBS_SCALARS[obs_type].values():
obs[regobs_attr] = 0
if isinstance(reg, list) and len(reg) > 0:
obs[regobs_attr] = reg[0][regobs_attr]
elif not isinstance(reg, list):
obs[regobs_attr] = reg[regobs_attr]
if obs[regobs_attr] is None:
obs[regobs_attr] = 0
except KeyError:
pass
if obs_type not in observations[key]:
observations[key][obs_type] = []
observations[key][obs_type].append(obs)
varsomdata_obses[key] += go.Observation(raw_obs).Observations
# We want the most competent observations first
for key, date_region in observations.items():
for reg_key, reg_type in date_region.items():
reg_type.sort(key=lambda x: x['competence'], reverse=True)
observations_neigh[key][reg_key] = reg_type.copy()
rng = np.random.default_rng(1984)
for (date, region), date_region in observations.items():
for reg_key in date_region.keys():
reg_neigh = []
for neighbour in rng.permutation(REGION_NEIGH[region]):
try:
reg_neigh += observations[(date, neighbour)][reg_key]
except KeyError:
pass
reg_neigh.sort(key=lambda x: x['competence'], reverse=True)
observations_neigh[(date, region)][reg_key] += reg_neigh
df_dict = {}
for key, observation in observations_neigh.items():
# Use 2 most competent observations, and list both categories as well as scalars
for obs_idx in range(0, 2):
# One type of observation (test, danger signs etc.) at a time
for regobs_type in req_set:
obses = observation[
regobs_type] if regobs_type in observation else []
# Go through each requested class attribute from the specified observation type
for attr, cat in REGOBS_CLASSES[regobs_type].items():
# We handle categories using 1-hot, so we step through each category
for cat_name in cat.values():
attr_name = f"regobs_{REG_ENG[regobs_type]}_{_camel_to_snake(attr)}_{cat_name}_{obs_idx}"
if attr_name not in df_dict:
df_dict[attr_name] = {}
df_dict[attr_name][key] = obses[obs_idx][
cat_name] if len(obses) > obs_idx else 0
# Go through all requested scalars
for attr, (regobs_attr,
conv) in REGOBS_SCALARS[regobs_type].items():
attr_name = f"regobs_{REG_ENG[regobs_type]}_{_camel_to_snake(attr)}_{obs_idx}"
if attr_name not in df_dict:
df_dict[attr_name] = {}
try:
df_dict[attr_name][key] = conv(
obses[obs_idx]
[regobs_attr]) if len(obses) > obs_idx else 0
except TypeError:
df_dict[attr_name][key] = 0
if "accuracy" not in df_dict:
df_dict["accuracy"] = {}
df_dict['accuracy'][key] = sum(
map(lambda x: {
0: 0,
1: 1,
2: -1,
3: -1
}[x['ForecastCorrectTID']], observation['Skredfarevurdering'])
) if 'Skredfarevurdering' in observation else 0
if avalanche_index:
if "regobs_avalancheidx" not in df_dict:
df_dict["regobs_avalancheidx"] = {}
avalanche_indices = list(
map(lambda x: x.index,
gm.get_avalanche_index(varsomdata_obses[key])))
if avalanche_indices:
df_dict['regobs_avalancheidx'][key] = max(avalanche_indices)
else:
df_dict['regobs_avalancheidx'][key] = 0
return df_dict
def make_season_calender_plots(year='2019-20',
plot_folder=env.plot_folder,
html_folder=env.output_folder + 'views/',
web_pickle_folder=env.output_folder +
'webpickles/'):
"""Makes observation calender plots for both observer and region for display on web page for the season 2018-19.
Method includes a request for list of relevant observers."""
from_year = int(year[0:4])
to_year = int('20' + year[-2:])
from_day = dt.date(from_year, 11, 1)
to_day = dt.date(to_year, 6, 30)
# if the seasons expected end is after todays date, set it to today.
if to_day > dt.date.today():
to_day = dt.date.today()
# list of months to be plotted
months = []
month = from_day
while month < to_day:
months.append(month)
almost_next = month + dt.timedelta(days=35)
month = dt.date(almost_next.year, almost_next.month, 1)
# Get all regions
region_ids = gm.get_forecast_regions(year)
# get a list of relevant observers to plot and make pickle for adding to the web-folder
all_observations_nest = gvp.get_all_observations(year,
output='List',
geohazard_tids=10)
all_observations_list = gvp.get_all_observations(year,
output='FlatList',
geohazard_tids=10)
observer_dict = {}
for o in all_observations_nest:
if o.ObserverID in observer_dict.keys():
observer_dict[o.ObserverID].add_one_observation_count()
else:
observer_dict[o.ObserverID] = ObserverData(o.ObserverID,
o.NickName,
observation_count_inn=1)
observer_list = []
observer_list_web = []
ordered_observer_dict = col.OrderedDict(
sorted(observer_dict.items(),
key=lambda t: t[1].observation_count,
reverse=True))
for k, v in ordered_observer_dict.items():
if v.observation_count > 4:
observer_list.append(
ObserverData(v.observer_id,
v.observer_nick,
observation_count_inn=v.observation_count))
observer_list_web.append(
[v.observer_id, v.observer_nick, v.observation_count])
if not os.path.exists(web_pickle_folder):
os.makedirs(web_pickle_folder)
mp.pickle_anything(observer_list_web,
'{0}observerlist.pickle'.format(web_pickle_folder))
# run the stuff
make_observer_plots(all_observations_list,
observer_list,
months,
plot_folder=plot_folder,
html_folder=html_folder)
make_region_plots(all_observations_list,
region_ids,
months,
plot_folder=plot_folder,
html_folder=html_folder)
make_svv_plots(all_observations_list,
observer_dict,
region_ids,
months,
plot_folder=plot_folder,
html_folder=html_folder)
plb.text(
x + 20, -270,
'*** {0} ganger er ett snoeskred med hoeysete index. \n'
' {1} som skredaktivitet og {2} med skjema for \n'
' enkeltskred.'.format(est_num_1, est_num_1_aval_act,
est_num_1_aval))
return
if __name__ == "__main__":
season = '2017-18'
### Get all regions
region_ids = gm.get_forecast_regions(season)
from_date, to_date = gm.get_forecast_dates(season)
# from_date, to_date = '2017-12-01', '2018-02-01'
# region_ids = [3014, 3015]
### get and make the data set
date_region, forecasted_dangers = step_1_make_data_set(
region_ids, from_date, to_date)
mp.pickle_anything([date_region, forecasted_dangers],
'{0}runforavalancheactivity_step_1.pickle'.format(
env.local_storage))
### Find the observaton of highest value pr region pr date
date_region, forecasted_dangers = mp.unpickle_anything(
'{0}runforavalancheactivity_step_1.pickle'.format(env.local_storage))
date_region = step_2_find_most_valued(date_region)
from varsomdata import getmisc as gm
from utilities import fencoding as fe
import setenvironment as env
__author__ = 'Ragnar Ekker'
years = ['2014-15', '2015-16', '2016-17', '2017-18']
forecast_problems = []
forecast_dangers = []
observed_dangers = []
observed_problems = []
for y in years:
# Get forecast data. Different region ids from year to year.
region_ids = gm.get_forecast_regions(year=y)
from_date, to_date = gm.get_forecast_dates(y)
forecast_problems += gp.get_forecasted_problems(region_ids,
from_date,
to_date,
lang_key=1)
forecast_dangers += gd.get_forecasted_dangers(region_ids,
from_date,
to_date,
lang_key=1)
# Get observed data. All older data in regObs have been mapped to new regions.
region_ids = gm.get_forecast_regions(year='2016-17')
from_date, to_date = gm.get_forecast_dates(y,
padding=dt.timedelta(days=20))
current_years_observed_dangers = gd.get_observed_dangers(region_ids,
plb.vlines(1200, 0, 650, lw=2, color='k')
plb.vlines(1800, 0, 650, lw=2, color='k')
plb.vlines(2400, 0, 650, lw=2, color='k')
plb.xlim(-100, 2410)
plb.ylim(-10, 700)
plb.axis('off')
plb.savefig(plot_file_name + file_ext)
plb.close()
return
if __name__ == "__main__":
regions = gm.get_forecast_regions(year='2016-17')
date_from = "2016-12-01"
date_to = "2017-06-01"
pickle_warnings_file_name = '{0}{1}'.format(
thisenv.local_storage, 'runForMatrix warnings.pickle')
pickle_data_set_file_name = '{0}{1}'.format(
thisenv.local_storage, 'runForMatrix data set.pickle')
pickle_m3_file_name = '{0}{1}'.format(thisenv.local_storage,
'runForMatix m3.pickle')
plot_m3_file_name = '{0}m3 {1}-{2}'.format(thisenv.plot_folder,
date_from[0:4], date_to[2:4])
pickle_m3_v2_file_name = '{0}{1}'.format(thisenv.local_storage,
'runForMatix m3.v2.pickle')
plot_m3_v2_file_name = '{0}m3 {1}-{2}.v2'.format(thisenv.plot_folder,
