def get_forecast_region_for_regid(reg_id):
"""Returns the forecast region used at a given place in a given season.
:param reg_id: [int] regid in regObs
:return: [int] ForecastRegionTID from regObs
[string] ForecastRegionName from regObs
[observation] The full observation on this regID
"""
region_id, region_name, observation = None, None, None
try:
observation = go.get_data_as_class(reg_ids=reg_id)
utm33x = observation[0].UTMEast
utm33y = observation[0].UTMNorth
date = observation[0].DtObsTime
season = get_season_from_date(date.date())
region_id, region_name = get_forecast_region_for_coordinate(
utm33x, utm33y, season)
except:
error_msg = sys.exc_info()[0]
ml.log_and_print(
'[error] getmisc.py -> get_forecast_region_for_regid: Exception on RegID={0}: {1}.'
.format(reg_id, error_msg))
return region_id, region_name, observation
def incident_troms_winter_2018_for_markus():
"""Communication dated 2018-11-29
Hei Ragnar og Jostein
Kan en av dere hjelpe meg å ta ut et plott som viser antall registrerte ulykker og hendelser i
varslingsregionene Tromsø, Lyngen, Sør-Troms og Indre-Troms for
perioden 15.02 – 15.05.
...
Er du interessert i det som ligger i registrert i
regObs eller det som er kvalitetssikkert data og ligger på varsom?
Skal du ha hendelser som har hatt konsekvens?
Skal hendelsene plottes i tid eller vises i kart?
...
Varsom
Ikke nødvendigvis konsekvens
Tid
:return:
"""
pickle_file_name = '{0}incident_troms_winter_2018_for_markus.pickle'.format(
env.local_storage)
from_date = dt.date(2018, 2, 15) # '2018-02-15'
to_date = dt.date(2018, 5, 15) # '2018-05-15'
# Tromsø, Lyngen, Sør-Troms og Indre-Troms
regions = [3011, 3010, 3012, 3013]
get_new = False
if get_new:
all_varsom_incidents = gm.get_varsom_incidents(
add_forecast_regions=True, add_observations=True)
all_regobs_avalobs_and_incidents = go.get_data_as_class(
from_date,
to_date,
registration_types=[11, 26],
region_ids=regions,
output='Nest')
mp.pickle_anything(
[all_varsom_incidents, all_regobs_avalobs_and_incidents],
pickle_file_name)
else:
[all_varsom_incidents, all_regobs_avalobs_and_incidents
] = mp.unpickle_anything(pickle_file_name)
varsom_incidents = mm.make_date_int_dict(start_date=from_date,
end_date=to_date)
regobs_avalobs_and_incidents = mm.make_date_int_dict(start_date=from_date,
end_date=to_date)
for i in all_varsom_incidents:
if from_date <= i.date <= to_date:
if i.region_id in regions:
if i.date in varsom_incidents.keys():
varsom_incidents[i.date] += 1
for i in all_regobs_avalobs_and_incidents:
if from_date <= i.DtObsTime.date() <= to_date:
if i.ForecastRegionTID in regions:
if i.DtObsTime.date() in regobs_avalobs_and_incidents.keys():
regobs_avalobs_and_incidents[i.DtObsTime.date()] += 1
sum_varsom = sum(varsom_incidents.values())
sum_regobs = sum(regobs_avalobs_and_incidents.values())
varsom_incident_troms_winter_2018_for_markus = '{0}varsom_incident_troms_winter_2018_for_markus.csv'.format(
env.output_folder)
regobs_incident_troms_winter_2018_for_markus = '{0}regobs_incident_troms_winter_2018_for_markus.csv'.format(
env.output_folder)
with open(varsom_incident_troms_winter_2018_for_markus,
'w',
encoding='utf-8') as f:
make_header = True
for k, v in varsom_incidents.items():
if make_header:
f.write('date; number\n')
make_header = False
f.write('{}; {}\n'.format(k, v))
with open(regobs_incident_troms_winter_2018_for_markus,
'w',
encoding='utf-8') as f:
make_header = True
for k, v in regobs_avalobs_and_incidents.items():
if make_header:
f.write('date; number\n')
make_header = False
f.write('{}; {}\n'.format(k, v))
pass
# If multiple id's wil be used, give them as list. Result may be returned as a list of forms (default) or
# nested, i.e. all forms are listed under their respective observations.
two_obs = go.get_data(reg_ids=[130548, 130328], output='Nest')
# A request may specify a time period and specific geohazards.
# Snow is 10 and ice is 70. Water is 60. Dirt is [20, 30, 40]
all_data_snow = go.get_data('2016-12-30', '2017-01-01', geohazard_tids=10)
ice_data = go.get_data(from_date='2016-10-01',
to_date='2016-11-01',
geohazard_tids=70,
output='Nest')
# The data may be returned as a list of classes, as opposed to the default return in get_data which are dictionaries
# raw as given on regObs webapi.
data_as_classes = go.get_data_as_class('2018-05-01', '2018-08-01')
# We may get observation forms directly. Note, from and to date are first and may also be given as
# positional arguments, even though Is recommend keyword arguments.
land_slides = go.get_land_slide_obs('2018-01-01', '2018-02-01')
incident = go.get_incident('2012-03-01', '2012-03-10')
ice_thicks = go.get_ice_thickness('2018-01-20', '2018-02-10')
snow_surface = go.get_snow_surface_observation('2018-01-28', '2018-02-01')
problems = go.get_avalanche_problem_2('2017-03-01', '2017-03-10')
# We may request an observation count.
# Remember: if forms are grouped under the observation, its a nest.
# If the forms are separate items in the list, its a list.
seasonal_count_regs = go.get_data('2016-08-01',
'2017-08-01',
output='Count nest')
def get_snow_obs(from_date, to_date):
all_data_snow = go.get_data_as_class(from_date, to_date, geohazard_tids=10)
return all_data_snow
def get_all_observations(year,
output='Nest',
geohazard_tids=None,
lang_key=1,
max_file_age=23):
"""Specialized method for getting all observations for one season (1. sept to 31. august).
For the current season (at the time of writing, 2018-19), if request has been made the last 23hrs,
data is retrieved from a locally stored pickle, if not, new request is made to the regObs api. Previous
seasons are not requested if a pickle is found in local storage.
:param year: [string] Eg. season '2017-18' (sept-sept) or one single year '2018'
:param output: [string] 'Nest' or 'List'
:param geohazard_tids: [int or list of ints] Default None gives all. Note, pickle stores all, but this option returns a select
:param lang_key [int] 1 is norwegian, 2 is english
:param max_file_age: [int] hrs how old the file is before new is retrieved
:return:
"""
from_date, to_date = gm.get_dates_from_season(year=year)
file_name_list = '{0}all_observations_list_{1}_lk{2}.pickle'.format(
env.local_storage, year, lang_key)
file_name_nest = '{0}all_observations_nest_{1}_lk{2}.pickle'.format(
env.local_storage, year, lang_key)
get_new = True
date_limit = dt.datetime.now() - dt.timedelta(hours=max_file_age)
# if we are well out of the current season (30 days) its little chance the data set has changed.
current_season = gm.get_season_from_date(dt.date.today() -
dt.timedelta(30))
if geohazard_tids:
if not isinstance(geohazard_tids, list):
geohazard_tids = [geohazard_tids]
if os.path.exists(file_name_list):
# if file contains a season long gone, dont make new.
if year == current_season:
file_age = dt.datetime.fromtimestamp(
os.path.getmtime(file_name_list))
# If file is newer than the given time limit, dont make new.
if file_age > date_limit:
# If file size larger than that of an nearly empty file, dont make new.
if os.path.getsize(file_name_list) > 100: # 100 bytes limit
get_new = False
else:
get_new = False
if get_new:
# When get new, get all geo hazards
nested_observations = go.get_data_as_class(from_date=from_date,
to_date=to_date,
output='Nest',
geohazard_tids=None,
lang_key=lang_key)
mp.pickle_anything(nested_observations, file_name_nest)
listed_observations = []
for d in nested_observations:
for o in d.Observations:
if _observation_is_not_empty(o):
listed_observations.append(o)
for p in d.Pictures:
# p['RegistrationName'] = 'Bilde'
listed_observations.append(p)
mp.pickle_anything(listed_observations, file_name_list)
if output == 'Nest':
all_nested_observations = mp.unpickle_anything(file_name_nest)
nested_observations = []
if geohazard_tids:
for o in all_nested_observations:
if o.GeoHazardTID in geohazard_tids:
nested_observations.append(o)
else:
nested_observations = all_nested_observations
return nested_observations
elif output == 'List':
all_listed_observations = mp.unpickle_anything(file_name_list)
listed_observations = []
if geohazard_tids:
for o in all_listed_observations:
if o.GeoHazardTID in geohazard_tids:
listed_observations.append(o)
else:
listed_observations = all_listed_observations
return listed_observations
else:
ml.log_and_print(
'[warning] getvarsompickles.py -> get_all_registrations: Unknown output option'
)
return []
def get_varsom_incidents(add_forecast_regions=False,
add_observations=False,
add_forecasts=False):
"""Returns the incidents shown on varsom.no in a list of VarsomIncident objects.
Data input is a utf-8 formatted csv file in input folder. Original file might have newlines and
semicolons (;) in the cells. These need to be removed before saving as csv.
:param add_forecast_regions: [bool] If true the regid is used to get coordinates and the forecast region at the
observation date is added. Note, if true, some time is to be expected getting data.
:param add_observations: [bool] If true the observation is added when looking up the region name. This
option is only taken into account if add_forecast_regions is true.
:param add_forecasts: [bool] If true the forecast at that time and place is added to the incident. This
option is only taken into account if add_forecast_regions is true.
"""
incidents_file = '{}varsomsineskredulykker.csv'.format(
env.varsom_incidents)
varsom_incidents = rf.read_csv_file(incidents_file, VarsomIncident)
# map incident to forecast region
if add_forecast_regions:
for i in varsom_incidents:
if i.regid == []:
ml.log_and_print(
"[warning] getmisc.py -> get_varsom_incidents: No regid on incident on {}. No forecast region found."
.format(i.date))
else:
region_id, region_name, observation = get_forecast_region_for_regid(
i.regid[0])
i.add_forecast_region(region_id, region_name)
print("regid {}: {}".format(i.regid[0], i.date))
if add_observations:
i.add_observation(observation[0])
if len(i.regid) > 1:
observations = go.get_data_as_class(
reg_ids=i.regid[1:])
for o in observations:
i.add_observation(o)
if add_forecasts:
years = ['2014-15', '2015-16', '2016-17',
'2017-18'] # the years with data
all_forecasts = []
for y in years:
region_ids = get_forecast_regions(year=y)
from_date, to_date = get_forecast_dates(y)
all_forecasts += gd.get_forecasted_dangers(
region_ids, from_date, to_date)
for i in varsom_incidents:
incident_date = i.date
incident_region_id = i.region_id
print("{}: {}".format(i.location, incident_date))
for f in all_forecasts:
forecast_date = f.date
forecast_region_id = f.region_regobs_id
if incident_date == forecast_date:
if incident_region_id == forecast_region_id:
i.add_forecast(f)
return varsom_incidents