def make_problems_for_BritSiv(): """Brit Siv ønsket oversikt over varslede skredprobelemr og faregrader for Indre Fjordane of Fjordane de to siste årene (2015-2017). """ output_filename = '{0}Skredproblemer Indre Fjordane for BritSiv.csv'.format( env.output_folder) pickle_file_name = '{0}runavalancheproblems_britsiv.pickle'.format( env.local_storage) get_new = False all_dangers = [] if get_new: # Get Fjordane 2015-16 region_id = 121 from_date, to_date = gm.get_forecast_dates('2015-16') all_dangers += gd.get_forecasted_dangers(region_id, from_date, to_date) # Get Indre fjordane 2016-17 region_id = 3027 from_date, to_date = gm.get_forecast_dates('2016-17') all_dangers += gd.get_forecasted_dangers(region_id, from_date, to_date) mp.pickle_anything(all_dangers, pickle_file_name) else: all_dangers = mp.unpickle_anything(pickle_file_name) all_problems = [] for d in all_dangers: all_problems += d.avalanche_problems all_problems.sort(key=lambda AvalancheProblem: AvalancheProblem.date) _save_problems(all_problems, output_filename)
def get_data(from_date, to_date, region_ids, pickle_file_name_1, get_new): """Time consuming and inefficient. Not proud.. :param from_date: :param to_date: :param region_ids: :param pickle_file_name_1: :param get_new: :return: """ if get_new: # get all data and save to pickle all_incidents = go.get_incident(from_date, to_date, region_ids=region_ids, geohazard_tids=10) all_forecasts = gd.get_forecasted_dangers(region_ids, from_date, to_date) mp.pickle_anything([all_forecasts, all_incidents], pickle_file_name_1) else: # load data from pickle all_forecasts, all_incidents = mp.unpickle_anything(pickle_file_name_1) return all_forecasts, all_incidents
def get_all_ofoten(): """Dangers and problems for Ofoten (former Narvik). Writes file to .csv""" get_new = True get_observations = False write_csv = True plot_dangerlevels_simple = False select_years = [ '2012-13', '2013-14', '2014-15', '2015-16', '2016-17', '2017-18' ] region_id_Narvik = 114 # Narvik used from 2012 until nov 2016 region_id_Ofoten = 3015 # Ofoten introduced in november 2016 warnings_pickle = '{0}allforecasteddangerlevels_Ofoten_201218.pickle'.format( env.local_storage) warnings_csv = '{0}Faregrader Ofoten 2012-18.csv'.format(env.output_folder) warnings_plot = '{0}Faregrader Ofoten 2012-18.png'.format( env.output_folder) if get_new: all_warnings = [] all_evaluations = [] for y in select_years: if y in ['2016-17', '2017-18']: region_id = region_id_Ofoten else: region_id = region_id_Narvik from_date, to_date = gm.get_forecast_dates(year=y) all_warnings += gd.get_forecasted_dangers(region_id, from_date, to_date) if get_observations: all_evaluations += go.get_avalanche_evaluation_3( from_date, to_date, region_id) mp.pickle_anything([all_warnings, all_evaluations], warnings_pickle) else: [all_warnings, all_evaluations] = mp.unpickle_anything(warnings_pickle) if write_csv: # write to csv files _save_danger_and_problem_to_file(all_warnings, warnings_csv) elif plot_dangerlevels_simple: # Make simple plot from_date = gm.get_forecast_dates(select_years[0])[0] to_date = gm.get_forecast_dates(select_years[-1])[1] _make_plot_dangerlevels_simple(all_warnings, all_evaluations, warnings_plot, from_date, to_date) else: print("No output selected") return all_warnings, all_evaluations
def make_forecasts_for_Christian(): """Christian Jaedicke ønsker oversikt over varsel og skredproblemer siste tre år i Narvik.""" pickle_file_name = '{0}forecasts_ofoten_christian.pickle'.format( env.local_storage) get_new = False all_dangers = [] if get_new: # Get Narvik 2014-15 and 2015-16 region_id = 114 from_date, to_date = gm.get_forecast_dates('2014-15') all_dangers += gd.get_forecasted_dangers(region_id, from_date, to_date) from_date, to_date = gm.get_forecast_dates('2015-16') all_dangers += gd.get_forecasted_dangers(region_id, from_date, to_date) # Get Indre fjordane 2016-17 region_id = 3015 from_date, to_date = gm.get_forecast_dates('2016-17') all_dangers += gd.get_forecasted_dangers(region_id, from_date, to_date) mp.pickle_anything(all_dangers, pickle_file_name) else: all_dangers = mp.unpickle_anything(pickle_file_name) output_forecast_problems = '{0}Varsel Ofoten for Christian.csv'.format( env.output_folder) import collections as coll # Write forecasts to file with open(output_forecast_problems, 'w', encoding='utf-8') as f: make_header = True for d in all_dangers: for p in d.avalanche_problems: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(p.date, '%Y-%m-%d')), ('Region id', p.region_regobs_id), ('Region', p.region_name), ('DL', p.danger_level), ('Danger level', p.danger_level_name), ('Problem order', p.order), ('Problem', p.problem), ('Cause/ weaklayer', p.cause_name), ('Type', p.aval_type), ('Size', p.aval_size), ('Trigger', p.aval_trigger), ('Probability', p.aval_probability), ('Distribution', p.aval_distribution) ]) if make_header: f.write( ' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n')
def get_kdv(x_kdv, get_new=False): """Imports a x_kdv view from regObs and returns a dictionary with <key, value> = <ID, Name> An x_kdv is requested from the regObs api if a pickle file newer than a week exists. :param x_kdv: [string] x_kdv view :return dict: {} x_kdv as a dictionary Ex of use: aval_cause_kdv = get_kdv('AvalCauseKDV') Ex of url for returning values for IceCoverKDV in norwegian: http://api.nve.no/hydrology/regobs/v0.9.4/OData.svc/ForecastRegionKDV?$filter=Langkey%20eq%201%20&$format=json """ kdv_file = '{0}{1}.pickle'.format(se.kdv_elements_folder, x_kdv) dict = {} if get_new: url = 'http://api.nve.no/hydrology/regobs/{0}/OData.svc/{1}?$filter=Langkey%20eq%201%20&$format=json'\ .format(se.odata_version, x_kdv) ml.log_and_print("getregobsdata -> get_kdv: Getting KDV from URL:{0}".format(url)) kdv = requests.get(url).json() for a in kdv['d']['results']: try: if 'AvalCauseKDV' in url and a['ID'] > 9 and a['ID'] < 26: # this table gets special treatment dict[a["ID"]] = a["Description"] else: dict[a["ID"]] = a["Name"] except (RuntimeError, TypeError, NameError): pass mp.pickle_anything(dict, kdv_file) else: if os.path.exists(kdv_file): # Useful to test if the file is old and if so make a new one max_file_age = 7 mtime = os.path.getmtime(kdv_file) last_modified_date = dt.datetime.fromtimestamp(mtime).date() date_limit = dt.datetime.now() - dt.timedelta(days=max_file_age) # If file older than date limit, request a new. if last_modified_date < date_limit.date(): dict = get_kdv(x_kdv, get_new=True) else: # ml.log_and_print("getregobsdata -> get_kdv: Getting KDV from pickle:{0}".format(kdv_file)) dict = mp.unpickle_anything(kdv_file, print_message=False) else: dict = get_kdv(x_kdv, get_new=True) return dict
def make_forecasts_for_Sander(): """2018 August: Hei igjen Ragnar. Har du statistikk på varsla faregrad over ein heil sesong for Noreg? Eit snitt. XX dagar med faregrad 1, XX dagar med faregrad 2, XX dagar med faregrad 3.... fordelt på XX varslingsdagar. :return: """ pickle_file_name = '{0}201808_avalanche_forecasts_sander.pickle'.format( env.local_storage) get_new = False all_dangers = [] if get_new: years = ['2012-13', '2013-14', '2014-15', '2015-16'] for y in years: from_date, to_date = gm.get_forecast_dates(y) region_ids = gm.get_forecast_regions(y, get_b_regions=True) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) years = ['2016-17', '2017-18'] for y in years: from_date, to_date = gm.get_forecast_dates(y) region_ids = gm.get_forecast_regions(y, get_b_regions=True) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) mp.pickle_anything(all_dangers, pickle_file_name) else: all_dangers = mp.unpickle_anything(pickle_file_name) output_forecast_problems = '{0}201808 Faregrader for Sander.txt'.format( env.output_folder) import pandas as pd all_dangers_dict = [] for a in all_dangers: all_dangers_dict.append(a.__dict__) col_names = list(all_dangers_dict[0].keys()) all_dangers_df = pd.DataFrame(all_dangers_dict, columns=col_names, index=range(0, len(all_dangers_dict), 1)) a = 1
def make_m3_figs(forecaster_dict, nick, path=''): """Makes m3 tables for each forecaster. Uses methods from the runmatrix module. :param forecaster_dict: :param nick: is how I can select relevant warnings for this forecaster :param product_folder: location where plots (endproduct) is saved :param project_folder: many files generated; make project folder in product folder :return: """ from varsomscripts import matrix as mx f = forecaster_dict[nick] # select only warnings for this forecaster one_forecaster_warnings = f.warnings # prepare dataset pickle_data_set_file_name = '{0}runforefollow data set {1}.pickle'.format( env.local_storage, f.observer_id) mx.pickle_data_set(one_forecaster_warnings, pickle_data_set_file_name, use_ikke_gitt=False) forecaster_data_set = mp.unpickle_anything(pickle_data_set_file_name) # prepare the m3 elementes (cell contents) pickle_m3_v2_file_name = '{0}runforefollow m3 {1}.pickle'.format( env.local_storage, f.observer_id) mx.pickle_M3(forecaster_data_set, 'matrixconfiguration.v2.csv', pickle_m3_v2_file_name) m3_v2_elements = mp.unpickle_anything(pickle_m3_v2_file_name) # plot plot_m3_v2_file_name = '{0}{1}_m3'.format(path, f.observer_id) mx.plot_m3_v2(m3_v2_elements, plot_m3_v2_file_name) return
def get_data(region_id, start_date, end_date, get_new=True): """Gets all the data needed in the plots and pickles it so that I don't need to do requests to make plots. :param region_id: [int] Region ID is an int as given i ForecastRegionKDV :param start_date: [string] Start date. :param end_date: [string] End date. :param get_new: [bool] If true, new data is requested. If false, a local pickle is used for data. :return problems, dangers, aval_indexes: """ file_name = "{3}plotdangerandproblem_region{0}_{1}{2}.pickle".format( region_id, start_date.strftime('%Y'), end_date.strftime('%y'), env.local_storage) if not get_new and not os.path.exists(file_name): get_new = True ml.log_and_print( "[info] {0}get_data: pickle missing, getting new data.".format( log_reference), print_it=True) if get_new: dangers = gd.get_all_dangers(region_id, start_date, end_date) # Early years don't have the avalanche problem we will be analyzing if start_date > dt.date(2014, 11, 1): problems = gp.get_all_problems(region_id, start_date, end_date, add_danger_level=False) else: problems = [] aval_indexes = gm.get_avalanche_index(start_date, end_date, region_ids=region_id) mp.pickle_anything([problems, dangers, aval_indexes], file_name) else: problems, dangers, aval_indexes = mp.unpickle_anything(file_name) return problems, dangers, aval_indexes
def _map_obs_to_old_regions(obs, make_new=True): picle_file_name = '{}all observations with old coords.pickle'.format(env.local_storage) if make_new: for o in obs: utm_n = o.UTMNorth utm_e = o.UTMEast date = o.DtObsTime year = gm.get_season_from_date(date) region_id, region_name = gm.get_forecast_region_for_coordinate(utm_e, utm_n, year) o.ForecastRegionName = region_name o.ForecastRegionTID = region_id mp.pickle_anything(obs, picle_file_name) return obs else: obs_old_coords = mp.unpickle_anything(picle_file_name) return obs_old_coords
def make_avalanche_problemes(): data_output_filename = '{0}Alle skredproblemer.csv'.format( env.output_folder) pickle_file_name = '{0}runavalancheproblems.pickle'.format( env.local_storage) get_new = True if get_new: region_ids = [118, 128, 117] from_date = dt.date(2012, 12, 31) to_date = dt.date(2015, 7, 1) data = gp.get_all_problems(region_ids, from_date, to_date) mp.pickle_anything(data, pickle_file_name) else: data = mp.unpickle_anything(pickle_file_name) _save_problems_simple(data, data_output_filename) return
def get_incident_list(all_incidents, all_forecasts, desired_damage_extent_kdv, pickle_file_name_2, make_new_incident_list): """Each row in the incident list contains Incident and Forecast objects where date and forecast region match AND where incidents match the damage extent we wish to study. :param all_incidents: :param all_forecasts: :param desired_damage_extent_kdv: :param pickle_file_name_2: :param make_new_incident_list: :return: """ if make_new_incident_list: incident_list = [] for incident in all_incidents: if incident.DamageExtentTID in desired_damage_extent_kdv.keys(): incident_list.append( IncidentAndForecasts(incident, all_forecasts)) mp.pickle_anything(incident_list, pickle_file_name_2) else: incident_list = mp.unpickle_anything(pickle_file_name_2) return incident_list
def _get_all_snow(get_new=False): file_name = '{}observations and forecasts 2012-17.pickle'.format( env.local_storage) if get_new: all_observations = go.get_all_registrations('2012-12-01', '2017-07-01', geohazard_tids=10) years = ['2012-13', '2013-14', '2014-15', '2015-16', '2016-17'] all_forecasts = [] for y in years: from_date, to_date = gm.get_forecast_dates(y) region_ids = gm.get_forecast_regions(y) all_forecasts += gfa.get_avalanche_warnings( region_ids, from_date, to_date) mp.pickle_anything([all_observations, all_forecasts], file_name) else: [all_observations, all_forecasts] = mp.unpickle_anything(file_name) return all_observations, all_forecasts
messages = we.test_for_missing_elements(Inflow_DOP_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOP = Inflow_DOP_inn def add_Inflow_Chla(self, Inflow_Chla_inn): messages = we.test_for_missing_elements(Inflow_Chla_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_Chla = Inflow_Chla_inn def add_Inflow_DOC(self, Inflow_DOC_inn): messages = we.test_for_missing_elements(Inflow_DOC_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOC = Inflow_DOC_inn if __name__ == "__main__": yesturday = (dt.date.today() - dt.timedelta(days=1)).strftime("%Y-%m-%d") #harvest_and_save_blindern('2000-01-01', yesturday) #harvest_and_save_nordnesfjelet('2014-08-01', yesturday) data = harvest_for_mylake_hakkloa('2013-04-01', '2015-10-01') mp.pickle_anything(data, data.output_file_path + '.pickle') data2 = mp.unpickle_anything('{0}HAK_input'.format(env.data_path) + '.pickle') mfd.write_mylake_inputfile(data2)
def calculate_and_plot9d_season(period='2018-19'): """Calculate ice columns for 9 days and make plots of all ice thickness for a given season or optionally 'Today'. The inner workings: 1.1 Retrieves ice thickness observations from regObs. If period is given as a season, all observations for this season will be requested. All previous plots and local storage will be deleted. 1.2 If period='Today' ice thickness observations from today will be requested and plotted. Older plots will be in the folder. Metadata dict will be merged. 2. Calculate the 9 day prognosis from the observation time and plots the result. 3. Make a metadata json for handling files on iskart.no. Only confirmed files in folder will be added to metadata json. :param period: [String] Default is current season (2017-18). :return: """ log_referance = 'calculateandplot.py -> calculate_and_plot9d_season' # File names regid_metadata_json = '{}regid_metadata.json'.format( se.ni_dogn_plots_folder) regid_metadata_pickle = '{}regid_metadata.pickle'.format(se.local_storage) if period == 'Today': ice_thicks = gro.get_ice_thickness_today() else: # Empty the 9dogn folder # for file in os.listdir(se.ni_dogn_plots_folder): # file_path = os.path.join(se.ni_dogn_plots_folder, file) # try: # if os.path.isfile(file_path): # os.unlink(file_path) # except OSError: # pass # remove pickle with metadata try: os.remove(regid_metadata_pickle) except OSError: pass # Get new observations ice_thicks = gro.get_ice_thickness_observations(period, reset_and_get_new=True) # Calculate and plot for k, v in ice_thicks.items(): # If file is missing, make it. If observation is older than 11 days it is based on gridded data for sure and no plot file needed. make_plot = False max_file_age = 11 date_limit = dt.datetime.now() - dt.timedelta(days=max_file_age) file_names = os.listdir(se.ni_dogn_plots_folder) plot_filename = '{0}.png'.format(k) if plot_filename not in file_names: make_plot = True else: if v.date.date() > date_limit.date(): make_plot = True if make_plot: try: calculate_and_plot9d_regid(k, plot_folder=se.ni_dogn_plots_folder, observed_ice=v) except: error_msg = sys.exc_info()[0] ml.log_and_print( "[Error] {} Error making plot for {} {}".format( log_referance, k, error_msg)) # Make json with metadata for using files on iskart.no. Load metadata from pickle if available and # new observations where a plot is available will be made. if not os.path.exists(regid_metadata_pickle): regid_metadata = {} else: regid_metadata = mp.unpickle_anything(regid_metadata_pickle) list_of_plots = os.listdir(se.ni_dogn_plots_folder) for k, v in ice_thicks.items(): # only add metadata on files that are in the folder if '{0}.png'.format(k) in list_of_plots: date = v.date.date() region_name = v.metadata['OriginalObject']['ForecastRegionName'] if not region_name: region_name = 'Ukjent region' x, y = v.metadata['UTMEast'], v.metadata['UTMNorth'] lake_id = v.metadata['LocationID'] lake_name = v.metadata['LocationName'] if not lake_name: lake_name = 'E{} N{}'.format(x, y) regid_metadata[k] = { 'RegionName': region_name, 'LakeID': lake_id, 'LakeName': lake_name, 'Date': '{}'.format(date) } mp.pickle_anything(regid_metadata, regid_metadata_pickle) json_string = json.dumps(regid_metadata, ensure_ascii=False).encode('utf-8') with open(regid_metadata_json, 'wb') as f: f.write(json_string)
def plot_season_for_all_regobs_locations(year='2018-19', calculate_new=False, get_new_obs=False, make_plots=False, delete_old_plots=False): """Method specialized for scheduled plotting for iskart.no. Method makes a season plot for all ObsLocations in regObs where we have a first ice date. It may take some time to plot. 250 lakes for a season and for each plot weather params are requested from the GTS. The workings of the method: 1. get all locations ids and belonging observations where we have first ice. 2.1 if calculate new, empty sesong folder and pickle in local storage and calculate (and make plots if requested). 2.2 Make metadata json for showing files on iskart.no 3. All calculations are compared to observed data in scatter plot. :param year: [String] Season for plotting. eg: '2016-17' :param calculate_new: [bool] Calculate new ice thicks. If false only make the seasonal scatter. :param get_new_obs: [bool] :param make_plots: [bool] If False all calculations are made, but only the scatter comparison against observatiosn is ploted :param delete_old_plots: [bool] If True all former plots and pickles are removed. """ pickle_file_name_and_path = '{0}all_calculated_ice_{1}.pickle'.format( se.local_storage, year) location_id_metadata_json = '{}location_id_metadata.json'.format( se.sesong_plots_folder) if calculate_new: if delete_old_plots: # Empty the sesong plot folder for file in os.listdir(se.sesong_plots_folder): file_path = os.path.join(se.sesong_plots_folder, file) try: if os.path.isfile(file_path): os.unlink(file_path) except OSError: pass # remove pickle old data - because we are getting new try: os.remove(pickle_file_name_and_path) except OSError: pass all_observations = gro.get_all_season_ice(year, get_new=get_new_obs) from_date, to_date = gm.get_dates_from_year(year) all_calculated = {} all_observed = {} location_id_metadata = {} for location_id, observed_ice in all_observations.items(): try: calculated, observed, plot_filename = _plot_season( location_id, from_date, to_date, observed_ice, make_plots=make_plots, plot_folder=se.sesong_plots_folder) all_calculated[location_id] = calculated all_observed[location_id] = observed except: error_msg = sys.exc_info()[0] ml.log_and_print( "[error] calculateandplot.py -> plot_season_for_all_regobs_locations: Error making plot for {}" .format(error_msg, location_id)) # Make the json with metadata needed for iskart.no. Add only if the plot was made and thus file exists. if os.path.isfile(se.sesong_plots_folder + plot_filename): region_name = observed_ice[0].metadata['OriginalObject'][ 'ForecastRegionName'] if not region_name: region_name = 'Ukjent region' lake_id = observed_ice[0].metadata['LocationID'] x, y = observed_ice[0].metadata['UTMEast'], observed_ice[ 0].metadata['UTMNorth'] lake_name = observed_ice[0].metadata['LocationName'] if not lake_name: lake_name = 'E{} N{}'.format(x, y) location_id_metadata[location_id] = { 'RegionName': region_name, 'LakeID': lake_id, 'LakeName': lake_name, 'PlotFileName': plot_filename } mp.pickle_anything([all_calculated, all_observed], pickle_file_name_and_path) try: json_string = json.dumps(location_id_metadata, ensure_ascii=False).encode('utf-8') with open(location_id_metadata_json, 'wb') as f: f.write(json_string) except: error_msg = sys.exc_info()[0] ml.log_and_print( "[error]calculateandplot.py -> plot_season_for_all_regobs_locations: Cant write json. {}" .format(error_msg)) else: [all_calculated, all_observed] = mp.unpickle_anything(pickle_file_name_and_path) try: pts.scatter_calculated_vs_observed(all_calculated, all_observed, year) except: error_msg = sys.exc_info()[0] ml.log_and_print( "[error] calculateandplot.py -> plot_season_for_all_regobs_locations: {}. Could not plot scatter {}." .format(error_msg, year))
def incident_troms_winter_2018_for_markus(): """Communication dated 2018-11-29 Hei Ragnar og Jostein Kan en av dere hjelpe meg å ta ut et plott som viser antall registrerte ulykker og hendelser i varslingsregionene Tromsø, Lyngen, Sør-Troms og Indre-Troms for perioden 15.02 – 15.05. ... Er du interessert i det som ligger i registrert i regObs eller det som er kvalitetssikkert data og ligger på varsom? Skal du ha hendelser som har hatt konsekvens? Skal hendelsene plottes i tid eller vises i kart? ... Varsom Ikke nødvendigvis konsekvens Tid :return: """ pickle_file_name = '{0}incident_troms_winter_2018_for_markus.pickle'.format( env.local_storage) from_date = dt.date(2018, 2, 15) # '2018-02-15' to_date = dt.date(2018, 5, 15) # '2018-05-15' # Tromsø, Lyngen, Sør-Troms og Indre-Troms regions = [3011, 3010, 3012, 3013] get_new = False if get_new: all_varsom_incidents = gm.get_varsom_incidents( add_forecast_regions=True, add_observations=True) all_regobs_avalobs_and_incidents = go.get_data_as_class( from_date, to_date, registration_types=[11, 26], region_ids=regions, output='Nest') mp.pickle_anything( [all_varsom_incidents, all_regobs_avalobs_and_incidents], pickle_file_name) else: [all_varsom_incidents, all_regobs_avalobs_and_incidents ] = mp.unpickle_anything(pickle_file_name) varsom_incidents = mm.make_date_int_dict(start_date=from_date, end_date=to_date) regobs_avalobs_and_incidents = mm.make_date_int_dict(start_date=from_date, end_date=to_date) for i in all_varsom_incidents: if from_date <= i.date <= to_date: if i.region_id in regions: if i.date in varsom_incidents.keys(): varsom_incidents[i.date] += 1 for i in all_regobs_avalobs_and_incidents: if from_date <= i.DtObsTime.date() <= to_date: if i.ForecastRegionTID in regions: if i.DtObsTime.date() in regobs_avalobs_and_incidents.keys(): regobs_avalobs_and_incidents[i.DtObsTime.date()] += 1 sum_varsom = sum(varsom_incidents.values()) sum_regobs = sum(regobs_avalobs_and_incidents.values()) varsom_incident_troms_winter_2018_for_markus = '{0}varsom_incident_troms_winter_2018_for_markus.csv'.format( env.output_folder) regobs_incident_troms_winter_2018_for_markus = '{0}regobs_incident_troms_winter_2018_for_markus.csv'.format( env.output_folder) with open(varsom_incident_troms_winter_2018_for_markus, 'w', encoding='utf-8') as f: make_header = True for k, v in varsom_incidents.items(): if make_header: f.write('date; number\n') make_header = False f.write('{}; {}\n'.format(k, v)) with open(regobs_incident_troms_winter_2018_for_markus, 'w', encoding='utf-8') as f: make_header = True for k, v in regobs_avalobs_and_incidents.items(): if make_header: f.write('date; number\n') make_header = False f.write('{}; {}\n'.format(k, v)) pass
def make_forecasts_at_incidents_for_sander(): """Lager csv med alle varsomhendelser sammen med faregrad og de aktuelle skredproblemene (svakt lag, skredtype og skredproblemnavnert). Der det er gjort en regObs observasjon med «hendelse/ulykke» skjema fylt ut har jeg også lagt på skadeomfangsvurderingen. August 2018: Hei Jostein. Som du veit skal eg skriva om: Skredulykker knytt til skredproblem Du snakka om at det var muleg å få ut data for dette frå NVE sin database. Kan du hjelpa meg med det? Mvh Sander """ pickle_file_name = '{0}dl_inci_sander.pickle'.format(env.local_storage) output_incident_and_dl = '{0}Hendelse og faregrad til Sander.csv'.format( env.output_folder) get_new = False if get_new: varsom_incidents = gm.get_varsom_incidents(add_forecast_regions=True, add_forecasts=True, add_observations=True) mp.pickle_anything(varsom_incidents, pickle_file_name) else: varsom_incidents = mp.unpickle_anything(pickle_file_name) incident_and_dl = [] for i in varsom_incidents: incident_date = i.date danger_level = None problem_1 = None problem_2 = None problem_3 = None avalanche_type_1 = None avalanche_type_2 = None avalanche_type_3 = None weak_layer_1 = None weak_layer_2 = None weak_layer_3 = None dato_regobs = None damage_extent = None if i.forecast: danger_level = i.forecast.danger_level for p in i.forecast.avalanche_problems: if p.order == 1: problem_1 = p.problem weak_layer_1 = p.cause_name avalanche_type_1 = p.aval_type if p.order == 2: problem_2 = p.problem weak_layer_2 = p.cause_name avalanche_type_2 = p.aval_type if p.order == 3: problem_3 = p.problem weak_layer_3 = p.cause_name avalanche_type_3 = p.aval_type if i.observations: dato_regobs = i.observations[0].DtObsTime.date() for obs in i.observations: for o in obs.Observations: if isinstance(o, go.Incident): damage_extent = o.DamageExtentName incident_and_dl.append({ 'Dato': incident_date, # 'Dato (regObs)': dato_regobs, 'Region': i.region_name, 'Kommune': i.municipality, 'Dødsfall': i.fatalities, 'Alvorsgrad': damage_extent, 'Involverte': i.people_involved, 'Aktivitet': i.activity, 'Faregrad': danger_level, 'Skredproblem 1': problem_1, 'Skredtype 1': avalanche_type_1, 'Svaktlag 1': weak_layer_1, 'Skredproblem 2': problem_2, 'Skredtype 2': avalanche_type_2, 'Svaktlag 2': weak_layer_2, 'Skredproblem 3': problem_3, 'Skredtype 3': avalanche_type_3, 'Svaktlag 3': weak_layer_3, 'Kommentar': i.comment, 'regObs': '{}'.format(i.regid) }) # Write observed problems to file with open(output_incident_and_dl, 'w', encoding='utf-8') as f: make_header = True for i in incident_and_dl: if make_header: f.write(' ;'.join([fe.make_str(d) for d in i.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in i.values()]).replace( '[', '').replace(']', '') + '\n')
def make_dl_incident_markus(): """ From the beginning of time: get all forecasts. and then get how many on dl 3. get all incidents, excpt elrapp, and all in back country all these, get all on days in regions of dl 3. get all with serious caracter on days and in regions on dl 3 :return: """ pickle_file_name = '{0}incident_on_dl3_for_markus.pickle'.format( env.local_storage) years = ['2012-13', '2013-14', '2014-15', '2015-16', '2016-17'] get_new = False all_dangers = [] all_incidents = [] if get_new: for y in years: # get forecast regions used this year from_date, to_date = gm.get_forecast_dates(y) # get incidents for this year and map to this years forecast regions this_year_incidents = go.get_incident(from_date, to_date, geohazard_tids=10) for i in this_year_incidents: utm33x = i.UTMEast utm33y = i.UTMNorth region_id, region_name = gm.get_forecast_region_for_coordinate( utm33x, utm33y, y) i.region_regobs_id = region_id i.region_name = region_name all_incidents += this_year_incidents # get regions and the forecasts used this year region_ids = gm.get_forecast_regions(y) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) # in the end, pickle it all mp.pickle_anything([all_dangers, all_incidents], pickle_file_name) else: [all_dangers, all_incidents] = mp.unpickle_anything(pickle_file_name) all_dl3 = [] for d in all_dangers: if d.danger_level == 3: all_dl3.append(d) all_back_country_incidents = [] for i in all_incidents: if 'drift@svv' not in i.NickName: # if activity influenced is backcounty og scooter # should probably include 100 which is non specified incidents # giving this dataset the observations not specified if i.ActivityInfluencedTID in [ 100, 110, 111, 112, 113, 114, 115, 116, 117, 130 ]: all_back_country_incidents.append(i) all_back_country_incidents_with_consequence = [] for i in all_back_country_incidents: # If damageextent is nestenulykke, personer skadet eller personer omkommet if i.DamageExtentTID > 28: all_back_country_incidents_with_consequence.append(i) # find incidents in regions on days with danger level 3 # find incidetns in region on day with dl3 all_back_country_incidents_on_region_dl3 = [] all_back_country_incidents_with_consequence_on_region_dl3 = [] for d in all_dl3: danger_date = d.date danger_region_id = d.region_regobs_id for i in all_back_country_incidents: incident_date = i.DtObsTime.date() incident_region_id = i.ForecastRegionTID if incident_date == danger_date and incident_region_id == danger_region_id: all_back_country_incidents_on_region_dl3.append(i) for i in all_back_country_incidents_with_consequence: incident_date = i.DtObsTime.date() incident_region_id = i.ForecastRegionTID if incident_date == danger_date and incident_region_id == danger_region_id: all_back_country_incidents_with_consequence_on_region_dl3.append( i) print('Totalt varsler laget siden tidenes morgen: {}'.format( len(all_dangers))) print('Totalt varsler på fg 3: {}'.format(len(all_dl3))) print('Totalt antall hendelser i baklandet: {}'.format( len(all_back_country_incidents))) print('Totalt antall hendelser i baklandet med konsekvens: {}'.format( len(all_back_country_incidents_with_consequence))) print( 'Totalt antall hendelser i baklandet i regioner på dager med fg3: {}'. format(len(all_back_country_incidents_on_region_dl3))) print( 'Totalt antall hendelser i baklandet i regioner på dager med fg3 med konsekvens: {}' .format( len(all_back_country_incidents_with_consequence_on_region_dl3))) return
def get_all_observations(year, output='Nest', geohazard_tids=None, lang_key=1, max_file_age=23): """Specialized method for getting all observations for one season (1. sept to 31. august). For the current season (at the time of writing, 2018-19), if request has been made the last 23hrs, data is retrieved from a locally stored pickle, if not, new request is made to the regObs api. Previous seasons are not requested if a pickle is found in local storage. :param year: [string] Eg. season '2017-18' (sept-sept) or one single year '2018' :param output: [string] 'Nest' or 'List' :param geohazard_tids: [int or list of ints] Default None gives all. Note, pickle stores all, but this option returns a select :param lang_key [int] 1 is norwegian, 2 is english :param max_file_age: [int] hrs how old the file is before new is retrieved :return: """ from_date, to_date = gm.get_dates_from_season(year=year) file_name_list = '{0}all_observations_list_{1}_lk{2}.pickle'.format( env.local_storage, year, lang_key) file_name_nest = '{0}all_observations_nest_{1}_lk{2}.pickle'.format( env.local_storage, year, lang_key) get_new = True date_limit = dt.datetime.now() - dt.timedelta(hours=max_file_age) # if we are well out of the current season (30 days) its little chance the data set has changed. current_season = gm.get_season_from_date(dt.date.today() - dt.timedelta(30)) if geohazard_tids: if not isinstance(geohazard_tids, list): geohazard_tids = [geohazard_tids] if os.path.exists(file_name_list): # if file contains a season long gone, dont make new. if year == current_season: file_age = dt.datetime.fromtimestamp( os.path.getmtime(file_name_list)) # If file is newer than the given time limit, dont make new. if file_age > date_limit: # If file size larger than that of an nearly empty file, dont make new. if os.path.getsize(file_name_list) > 100: # 100 bytes limit get_new = False else: get_new = False if get_new: # When get new, get all geo hazards nested_observations = go.get_data_as_class(from_date=from_date, to_date=to_date, output='Nest', geohazard_tids=None, lang_key=lang_key) mp.pickle_anything(nested_observations, file_name_nest) listed_observations = [] for d in nested_observations: for o in d.Observations: if _observation_is_not_empty(o): listed_observations.append(o) for p in d.Pictures: # p['RegistrationName'] = 'Bilde' listed_observations.append(p) mp.pickle_anything(listed_observations, file_name_list) if output == 'Nest': all_nested_observations = mp.unpickle_anything(file_name_nest) nested_observations = [] if geohazard_tids: for o in all_nested_observations: if o.GeoHazardTID in geohazard_tids: nested_observations.append(o) else: nested_observations = all_nested_observations return nested_observations elif output == 'List': all_listed_observations = mp.unpickle_anything(file_name_list) listed_observations = [] if geohazard_tids: for o in all_listed_observations: if o.GeoHazardTID in geohazard_tids: listed_observations.append(o) else: listed_observations = all_listed_observations return listed_observations else: ml.log_and_print( '[warning] getvarsompickles.py -> get_all_registrations: Unknown output option' ) return []
def make_forecasts_for_Heidi(): """July 2018: Make list of avalanche forecasts for regions Voss, Svartisen og Fauske (and those before them) for Heidi Bjordal SVV""" pickle_file_name = '{0}201807_avalanche_forecasts_heidi.pickle'.format( env.local_storage) get_new = False all_dangers = [] if get_new: # Get Voss. ForecastRegionTID 124 form 2012-2016 and 3031 since. # Get Svartisen. ForecastRegionTID 131 form 2012-2016 and 3017 since. # Get Salten. ForecastRegionTID 133 form 2012-2016 and 3016 since. years = ['2012-13', '2013-14', '2014-15', '2015-16'] region_ids = [124, 131, 133] for y in years: from_date, to_date = gm.get_forecast_dates(y) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) years = ['2016-17', '2017-18'] region_ids = [3031, 3017, 3016] for y in years: from_date, to_date = gm.get_forecast_dates(y) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) mp.pickle_anything(all_dangers, pickle_file_name) else: all_dangers = mp.unpickle_anything(pickle_file_name) output_forecast_problems = '{0}201807 Snøskredvarsel for Heidi.txt'.format( env.output_folder) import collections as coll # Write forecasts to file with open(output_forecast_problems, 'w', encoding='utf-8') as f: make_header = True for d in all_dangers: for p in d.avalanche_problems: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(p.date, '%Y-%m-%d')), ('Region id', p.region_regobs_id), ('Region', p.region_name), ('DL', p.danger_level), ('Danger level', p.danger_level_name), ('Problem order', p.order), ('Problem', p.problem), ('Cause/ weaklayer', p.cause_name), ('Type', p.aval_type), ('Size', p.aval_size), ('Trigger', p.aval_trigger), ('Probability', p.aval_probability), ('Distribution', p.aval_distribution) ]) if make_header: f.write( ' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n') pass
def make_avalanche_problemes_for_techel(): """Gets forecastes and observed avalanche problems and dangers for Frank Techel. Takes 20-30 min to run a year. :return: """ pickle_file_name = '{0}runavalancheproblems_techel.pickle'.format( env.local_storage) years = ['2014-15', '2015-16', '2016-17', '2017-18'] get_new = False if get_new: forecast_problems = [] forecast_dangers = [] observed_dangers = [] observed_problems = [] for y in years: # Get forecast data. Different region ids from year to year. region_ids = gm.get_forecast_regions(year=y) from_date, to_date = gm.get_forecast_dates(y) forecast_problems += gp.get_forecasted_problems(region_ids, from_date, to_date, lang_key=2) forecast_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date, lang_key=2) # Get observed data. All older data in regObs have been mapped to new regions. region_ids = gm.get_forecast_regions(year='2016-17') from_date, to_date = gm.get_forecast_dates( y, padding=dt.timedelta(days=20)) this_years_observed_dangers = gd.get_observed_dangers(region_ids, from_date, to_date, lang_key=2) this_years_observed_problems = gp.get_observed_problems(region_ids, from_date, to_date, lang_key=2) # Update observations with forecast region ids and names used the respective years for od in this_years_observed_dangers: utm33x = od.metadata['Original data'].UTMEast utm33y = od.metadata['Original data'].UTMNorth region_id, region_name = gm.get_forecast_region_for_coordinate( utm33x, utm33y, y) od.region_regobs_id = region_id od.region_name = region_name for op in this_years_observed_problems: utm33x = op.metadata['Original data']['UtmEast'] utm33y = op.metadata['Original data']['UtmNorth'] region_id, region_name = gm.get_forecast_region_for_coordinate( utm33x, utm33y, y) op.region_regobs_id = region_id op.region_name = region_name observed_dangers += this_years_observed_dangers observed_problems += this_years_observed_problems mp.pickle_anything([ forecast_problems, forecast_dangers, observed_dangers, observed_problems ], pickle_file_name) else: [ forecast_problems, forecast_dangers, observed_dangers, observed_problems ] = mp.unpickle_anything(pickle_file_name) # Run EAWS mapping on all problems for p in forecast_problems: p.map_to_eaws_problems() for p in observed_problems: p.map_to_eaws_problems() output_forecast_problems = '{0}Techel forecast problems.csv'.format( env.output_folder) output_forecast_dangers = '{0}Techel forecast dangers.csv'.format( env.output_folder) output_observed_problems = '{0}Techel observed problems.csv'.format( env.output_folder) output_observed_dangers = '{0}Techel observed dangers.csv'.format( env.output_folder) import collections as coll # Write observed dangers to file with open(output_observed_dangers, 'w', encoding='utf-8') as f: make_header = True for d in observed_dangers: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(d.date, '%Y-%m-%d')), ('Reg time', dt.datetime.strftime(d.registration_time, '%Y-%m-%d %H:%M')), ('Region id', d.region_regobs_id), ('Region', d.region_name), ('Municipal', d.municipal_name), ('Nick', d.nick), ('Competence', d.competence_level), ('DL', d.danger_level), ('Danger level', d.danger_level_name), ('Forecast correct', d.forecast_correct), # ('Table', d.data_table), # ('URL', d.url), ]) if make_header: f.write(' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n') # Write forecasted dangers to file with open(output_forecast_dangers, 'w', encoding='utf-8') as f: make_header = True for d in forecast_dangers: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(d.date, '%Y-%m-%d')), ('Region id', d.region_regobs_id), ('Region', d.region_name), ('Nick', d.nick), ('DL', d.danger_level), ('Danger level', d.danger_level_name), # ('Table', d.data_table), # ('URL', d.url), ('Main message', ' '.join(d.main_message_en.split())) ]) if make_header: f.write(' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n') # Write forecasted problems to file with open(output_forecast_problems, 'w', encoding='utf-8') as f: make_header = True for p in forecast_problems: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(p.date, '%Y-%m-%d')), ('Region id', p.region_regobs_id), ('Region', p.region_name), ('Nick', p.nick_name), ('Problem order', p.order), ('Problem', p.problem), ('EAWS problem', p.eaws_problem), ('Cause/ weaklayer', p.cause_name), # ('TypeTID', p.aval_type_tid), ('Type', p.aval_type), ('Size', p.aval_size), ('Trigger', p.aval_trigger), ('Probability', p.aval_probability), ('Distribution', p.aval_distribution), ('DL', p.danger_level), ('Danger level', p.danger_level_name), # ('Table', p.regobs_table), # ('URL', p.url) ]) if make_header: f.write(' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n') # Write observed problems to file with open(output_observed_problems, 'w', encoding='utf-8') as f: make_header = True for p in observed_problems: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(p.date, '%Y-%m-%d')), ('Reg time', dt.datetime.strftime(p.registration_time, '%Y-%m-%d %H:%M')), ('Region id', p.region_regobs_id), ('Region', p.region_name), ('Municipal', p.municipal_name), ('Nick', p.nick_name), ('Competence', p.competence_level), # ('Problem order', p.order), ('EAWS problem', p.eaws_problem), ('Cause/ weaklayer', p.cause_name), # ('TypeTID', p.aval_type_tid), ('Type', p.aval_type), ('Catch 1', p.cause_attribute_crystal), ('Catch 2', p.cause_attribute_light), ('Catch 3', p.cause_attribute_soft), ('Catch 4', p.cause_attribute_thin), ('Size', p.aval_size), ('Trigger', p.aval_trigger), # ('Probability', p.aval_probability), # ('Distribution', p.aval_distribution), # ('RegID', p.regid), # ('Table', p.regobs_table), # ('URL', p.url) ]) if make_header: f.write(' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n')
def make_forecasts_at_incidents_for_mala(get_new=False): """Lager csv med alle varsomhendelser sammen med faregrad og de aktuelle skredproblemene (svakt lag, skredtype og skredproblemnavnert). Der det er gjort en regObs observasjon med «hendelse/ulykke» skjema fylt ut har jeg også lagt på skadeomfangsvurderingen. """ pickle_file_name = '{0}dl_inci_mala.pickle'.format(env.local_storage) output_incident_and_dl = '{0}incidents_mala.csv'.format(env.output_folder) if get_new: varsom_incidents = gm.get_varsom_incidents(add_forecast_regions=True, add_forecasts=True, add_observations=False) mp.pickle_anything(varsom_incidents, pickle_file_name) else: varsom_incidents = mp.unpickle_anything(pickle_file_name) incident_and_dl = [] for i in varsom_incidents: incident_date = i.date danger_level = None problem_1 = None problem_2 = None problem_3 = None avalanche_type_1 = None avalanche_type_2 = None avalanche_type_3 = None weak_layer_1 = None weak_layer_2 = None weak_layer_3 = None dato_regobs = None damage_extent = None if i.forecast: danger_level = i.forecast.danger_level for p in i.forecast.avalanche_problems: if p.order == 1: problem_1 = p.problem weak_layer_1 = p.cause_name avalanche_type_1 = p.aval_type if p.order == 2: problem_2 = p.problem weak_layer_2 = p.cause_name avalanche_type_2 = p.aval_type if p.order == 3: problem_3 = p.problem weak_layer_3 = p.cause_name avalanche_type_3 = p.aval_type if i.observations: dato_regobs = i.observations[0].DtObsTime.date() for obs in i.observations: for o in obs.Observations: if isinstance(o, go.Incident): damage_extent = o.DamageExtentName incident_and_dl.append({ 'Date': incident_date, # 'Dato (regObs)': dato_regobs, 'Region_id': i.region_id, 'Region': i.region_name, 'Fatalities': i.fatalities, 'Damage_extent': damage_extent, 'People_involved': i.people_involved, 'Activity': i.activity, 'Danger_level': danger_level, 'Avalanche_problem_1': problem_1, 'Avalanche_type_1': avalanche_type_1, 'Weak_layer_1': weak_layer_1, 'Avalanche_problem_2': problem_2, 'Avalanche_type_2': avalanche_type_2, 'Weak_layer_2': weak_layer_2, 'Avalanche_problem_3': problem_3, 'Avalanche_type_3': avalanche_type_3, 'Weak_layer_3': weak_layer_3, 'Comment': i.comment, 'regObs_id': '{}'.format(i.regid) }) # Write observed problems to file with open(output_incident_and_dl, 'w', encoding='utf-8') as f: make_header = True for i in incident_and_dl: if make_header: f.write(';'.join([fe.make_str(d) for d in i.keys()]) + '\n') make_header = False f.write(';'.join([fe.make_str(d) for d in i.values()]).replace( '[', '').replace(']', '') + '\n')
def get_all_season_ice(year, get_new=True): """Returns observed ice columns from regObs-webapi over a requested season. Ice covers representing first ice or ice cover lost are represented by an ice column of zero height. The workings of this routine: 1. Get one season of data from regobs-api, spreads them out to a long list. 2. Pick out only cover and column and group them on location_ids. We keep only locations with date for fist ice that season. All observations are mapped to the cover and column class in Ice.py. 3. Map all covers where first_ice or ice_cover_lost is True to zero-height columns. Remove the rest. If get_new=True new data is retrieved. If get_new=false data is picked from pickle. :param year: :param get_new: :return: """ file_name_and_path = '{0}get_all_season_ice_{1}.pickle'.format(se.local_storage, year) from_date, to_date = gm.get_dates_from_year(year) if get_new: all_observations = get_data(from_date=from_date, to_date=to_date, geohazard_tids=70) all_locations = {} for o in all_observations: if o['RegistrationTid'] == 51 or o['RegistrationTid'] == 50: if o['LocationId'] in all_locations.keys(): all_locations[o['LocationId']].append(o) else: all_locations[o['LocationId']] = [o] # sort oldest first on each location for l, obs in all_locations.items(): sorted_list = sorted(obs, key=lambda d: d['DtObsTime']) all_locations[l] = sorted_list # Use only locations with verified "first ice cover" date. all_locations_with_first_ice = {} for l, obs in all_locations.items(): for o in obs: if o['RegistrationTid'] == 51: # if the ice cover is partly or fully formed on observation location or the lake # 2) delvis islagt på målestedet # 3) helt islagt på målestedet # 21) hele sjøen islagt if (o['FullObject']['IceCoverTID'] == 2) or (o['FullObject']['IceCoverTID'] == 3) or \ (o['FullObject']['IceCoverTID'] == 21): # and if ice cover before was # 1) isfritt på målestedet # 2) delvis islagt på målestedet, # 11) islegging langs land # 20) hele sjøen isfri, this is fist ice if (o['FullObject']['IceCoverBeforeTID'] == 1) or (o['FullObject']['IceCoverBeforeTID'] == 2) or \ (o['FullObject']['IceCoverBeforeTID'] == 11) or (o['FullObject']['IceCoverBeforeTID'] == 20): all_locations_with_first_ice[l] = obs # Map all observations from regObs-webapi result structure to the classes in ice.py all_locations_with_classes = {} for l, obs in all_locations_with_first_ice.items(): all_locations_with_classes[l] = [] location_name = obs[0]['LocationName'] previous_cover = ice.IceCover(dt.datetime.strptime(from_date, "%Y-%m-%d").date(), "Ikke gitt", 'Ikke gitt', location_name) for o in obs: if o['RegistrationTid'] == 51: cover_date = dt.datetime.strptime(o['DtObsTime'][0:16], "%Y-%m-%dT%H:%M") cover = o['FullObject']['IceCoverTName'] cover_before = o['FullObject']['IceCoverBeforeTName'] cover_after = o['FullObject']['IceCoverAfterTName'] cover_tid = o['FullObject']['IceCoverTID'] cover_before_tid = o['FullObject']['IceCoverBeforeTID'] cover_after_tid = o['FullObject']['IceCoverAfterTID'] this_cover = ice.IceCover(cover_date, cover, cover_before, location_name) this_cover.set_regid(o['RegId']) this_cover.set_locationid(o['LocationId']) this_cover.set_utm(o['UtmNorth'], o['UtmEast'], o['UtmZone']) this_cover.set_cover_after(cover_after, cover_after_tid) this_cover.add_original_object(o) # if the ice cover is partly or fully formed on observation location or the lake # 2) delvis islagt på målestedet # 3) helt islagt på målestedet # 21) hele sjøen islagt if cover_tid == 2 or cover_tid == 3 or cover_tid == 21: # and if ice cover before was # 1) isfritt, nå første is på målestedet på målestedet # 2) isfritt, nå første is ved land # 4) Gradvis islegging if cover_before_tid == 1 or cover_before_tid == 2 or cover_before_tid == 4: this_cover.mark_as_first_ice() # if the ice cover is partly or fully gone on location and there was ice yesterday # 1) Isfritt på målestedet # 2) delvis islagt på målestedet # 20) Hele sjøen isfri if cover_tid == 1 or cover_tid == 2 or cover_tid == 20: # 10) isfritt resten av vinteren # Accepts also ice free observation after 15. March to_year = this_cover.date.year first_accepted_date = dt.datetime(to_year, 3, 15) last_accepted_date = dt.datetime(to_year, 9, 1) if cover_after_tid == 10 or (cover_date > first_accepted_date and cover_date < last_accepted_date): this_cover.mark_as_ice_cover_lost() # copy of this cover so that in next iteration I may look up previous cover. previous_cover = cp.deepcopy(this_cover) all_locations_with_classes[l].append(this_cover) if o['RegistrationTid'] == 50: ice_column = _webapi_ice_col_to_ice_class(o) if ice_column is not None: all_locations_with_classes[l].append(ice_column) # Map all covers where first_ice or ice_cover_lost is True to zero-height columns. Remove all the rest. all_locations_with_columns = {} for k, v in all_locations_with_classes.items(): new_v = [] for o in v: if isinstance(o, ice.IceCover): if o.first_ice or o.ice_cover_lost: new_o = ice.IceColumn(o.date, []) new_o.add_metadata('OriginalObject', o.metadata['OriginalObject']) new_o.add_metadata('UTMEast', o.metadata['UTMEast']) new_o.add_metadata('UTMNorth', o.metadata['UTMNorth']) new_o.add_metadata('UTMZone', o.metadata['UTMZone']) new_o.add_metadata('LocationName', o.locationName) new_o.add_metadata('LocationID', o.LocationID) new_v.append(new_o) else: new_v.append(o) all_locations_with_columns[k] = new_v mp.pickle_anything(all_locations_with_columns, file_name_and_path) else: # if pickle file with all data for the season does not exist, get data anyway if not os.path.exists(file_name_and_path): all_locations_with_columns = get_all_season_ice(year, get_new=True) else: all_locations_with_columns = mp.unpickle_anything(file_name_and_path, print_message=False) return all_locations_with_columns
def get_node_list(pickle_file_name_3, make_new_node_list, desired_damage_extent_kdv, incident_list): """Makes a list of NodesAndValues objects. All nodes get an object and relations between the nodes are calculated. Lots of looping. :param pickle_file_name_3: :param make_new_node_list: :param desired_damage_extent_kdv :param incident_list :return: """ if make_new_node_list: problem_kdv = { 0: 'Ikke gitt', 3: 'Toerre loessnoeskred', 5: 'Vaate loessnoeskred', 7: 'Nysnoeflak', 10: 'Fokksnoe', 20: 'Nysnoe', 30: 'Vedvarende svakt lag', 37: 'Dypt vedvarende svakt lag', 40: 'Vaat snoe', 45: 'Vaate flakskred', 50: 'Glideskred' } cause_kdv = gkdv.get_kdv('AvalCauseKDV') danger_kdv = gkdv.get_kdv('AvalancheDangerKDV') activity_influenced_kdv = gkdv.get_kdv('ActivityInfluencedKDV') nodes_dict = {} id_counter = -1 for cause_tid, cause_kdve in cause_kdv.items(): cause_name = cause_kdve.Name if 'kke gitt' in cause_name: cause_name = 'Svakt lag {0}'.format(cause_name) if cause_kdve.IsActive: id_counter += 1 nodes_dict[cause_name] = id_counter for problem_tid, problem_name in problem_kdv.items(): if 'kke gitt' in problem_name: problem_name = 'Skredproblem {0}'.format(problem_name) id_counter += 1 nodes_dict[problem_name] = id_counter for desired_damage_extent_tid, desired_damage_extent_name in desired_damage_extent_kdv.items( ): if 'kke gitt' in desired_damage_extent_name: desired_damage_extent_name = 'Skadeomfang {0}'.format( desired_damage_extent_name) id_counter += 1 nodes_dict[desired_damage_extent_name] = id_counter for activity_influenced_tid, activity_influenced_kdve in activity_influenced_kdv.items( ): if activity_influenced_tid < 200: # only snow activity_influenced_name = activity_influenced_kdve.Name if 'kke gitt' in activity_influenced_name: activity_influenced_name = 'Aktivitet {0}'.format( activity_influenced_name) if activity_influenced_kdve.IsActive: id_counter += 1 nodes_dict[activity_influenced_name] = id_counter for danger_tid, danger_kdve in danger_kdv.items(): danger_name = danger_kdve.Name if 'kke gitt' in danger_name: 'Faregrad {0}'.format(danger_name) if danger_kdve.IsActive: id_counter += 1 nodes_dict[danger_name] = id_counter make_nodes = True nodes_and_values = [] print_counter = 0 for i in incident_list: print('Index {0} of 192 in incidentlist'.format(print_counter)) print_counter += 1 if i.forecast: cause = i.forecast.avalanche_problems[0].cause_name if 'kke gitt' in cause: cause = 'Svakt lag {0}'.format(cause) problem = i.forecast.avalanche_problems[0].main_cause if 'kke gitt' in problem: problem = 'Skredproblem {0}'.format(problem) # Loop through the cause and problem list. # If it is the first run make the nodes. # If the causes in the lists match what is in the list of actual incidents, add one to the node. for cause_tid, cause_kdve in cause_kdv.items(): if cause_kdve.IsActive: cause_name = cause_kdve.Name if 'kke gitt' in cause_name: cause_name = 'Svakt lag {0}'.format(cause_name) for problem_tid, problem_name in problem_kdv.items(): if 'kke gitt' in problem_name: problem_name = 'Skredproblem {0}'.format( problem_name) if make_nodes: # the run of the first item of incident_list covers all nodes nodes_and_values.append( NodesAndValues(cause_name, nodes_dict[cause_name], problem_name, nodes_dict[problem_name])) if cause in cause_name and problem in problem_name: for nv in nodes_and_values: if cause in nv.node_name and problem in nv.target_name: nv.add_one() damage_extent = i.incident.DamageExtentName if 'kke gitt' in damage_extent: damage_extent = 'Skadeomfang {0}'.format(damage_extent) for problem_tid, problem_name in problem_kdv.items(): if 'kke gitt' in problem_name: problem_name = 'Skredproblem {0}'.format(problem_name) for desired_damage_extent_tid, desired_damage_extent_name in desired_damage_extent_kdv.items( ): if 'kke gitt' in desired_damage_extent_name: desired_damage_extent_name = 'Skadeomfang {0}'.format( desired_damage_extent_name) if make_nodes: nodes_and_values.append( NodesAndValues( problem_name, nodes_dict[problem_name], desired_damage_extent_name, nodes_dict[desired_damage_extent_name])) if problem in problem_name and damage_extent in desired_damage_extent_name: for nv in nodes_and_values: if problem in nv.node_name and damage_extent in nv.target_name: nv.add_one() activity_influenced = i.incident.ActivityInfluencedName if 'kke gitt' in activity_influenced: activity_influenced = 'Aktivitet {0}'.format( activity_influenced) for desired_damage_extent_tid, desired_damage_extent_name in desired_damage_extent_kdv.items( ): if 'kke gitt' in desired_damage_extent_name: desired_damage_extent_name = 'Skadeomfang {0}'.format( desired_damage_extent_name) for activity_influenced_tid, activity_influenced_kdve in activity_influenced_kdv.items( ): if activity_influenced_tid < 200: # only snow activity_influenced_name = activity_influenced_kdve.Name if 'kke gitt' in activity_influenced_name: activity_influenced_name = 'Aktivitet {0}'.format( activity_influenced_name) if activity_influenced_kdve.IsActive: if make_nodes: nodes_and_values.append( NodesAndValues( desired_damage_extent_name, nodes_dict[ desired_damage_extent_name], activity_influenced_name, nodes_dict[ activity_influenced_name])) if desired_damage_extent_name in damage_extent and activity_influenced_name in activity_influenced: for nv in nodes_and_values: if desired_damage_extent_name in nv.node_name and activity_influenced_name in nv.target_name: nv.add_one() danger = i.forecast.danger_level_name if 'kke gitt' in danger: danger = 'Faregrad {0}'.format(danger) for activity_influenced_tid, activity_influenced_kdve in activity_influenced_kdv.items( ): if activity_influenced_tid < 200: activity_influenced_name = activity_influenced_kdve.Name if 'kke gitt' in activity_influenced_name: activity_influenced_name = 'Aktivitet {0}'.format( activity_influenced_name) if activity_influenced_kdve.IsActive: for danger_tid, danger_kdve in danger_kdv.items(): danger_name = danger_kdve.Name if 'kke gitt' in danger_name: 'Faregrad {0}'.format(danger_name) if danger_kdve.IsActive: if make_nodes: nodes_and_values.append( NodesAndValues( activity_influenced_name, nodes_dict[ activity_influenced_name], danger_name, nodes_dict[danger_name])) if activity_influenced_name in activity_influenced and danger_name in danger: for nv in nodes_and_values: if activity_influenced_name in nv.node_name and danger_name in nv.target_name: nv.add_one() make_nodes = False mp.pickle_anything(nodes_and_values, pickle_file_name_3) else: nodes_and_values = mp.unpickle_anything(pickle_file_name_3) return nodes_and_values
### Get all regions region_ids = gm.get_forecast_regions(season) from_date, to_date = gm.get_forecast_dates(season) # from_date, to_date = '2017-12-01', '2018-02-01' # region_ids = [3014, 3015] ### get and make the data set date_region, forecasted_dangers = step_1_make_data_set( region_ids, from_date, to_date) mp.pickle_anything([date_region, forecasted_dangers], '{0}runforavalancheactivity_step_1.pickle'.format( env.local_storage)) ### Find the observaton of highest value pr region pr date date_region, forecasted_dangers = mp.unpickle_anything( '{0}runforavalancheactivity_step_1.pickle'.format(env.local_storage)) date_region = step_2_find_most_valued(date_region) mp.pickle_anything([date_region, forecasted_dangers], '{0}runforavalancheactivity_step_2.pickle'.format( env.local_storage)) ### ready to add to count elements date_region, forecasted_dangers = mp.unpickle_anything( '{0}runforavalancheactivity_step_2.pickle'.format(env.local_storage)) elements = rf.read_configuration_file( '{0}aval_dl_configuration.csv'.format(env.matrix_configurations), ActivityAndDanger) elements = step_3_count_occurances(date_region, elements) mp.pickle_anything([date_region, forecasted_dangers, elements], '{0}runforavalancheactivity_step_3.pickle'.format( env.local_storage))
def get_all_forecasts(year, lang_key=1, max_file_age=23): """Specialized method for getting all forecasts for one season. For the current season (at the time of writing, 2018-19), if a request has been made the last 23hrs, data is retrieved from a locally stored pickle, if not, new request is made to the regObs api. Previous seasons are not requested if a pickle is found in local storage. :param year: [string] Eg. season '2017-18' :param lang_key [int] 1 is norwegian, 2 is english :param max_file_age: [int] hrs how old the file is before new is retrieved :return valid_forecasts: [list of AvalancheWarning] """ from_date, to_date = gm.get_forecast_dates(year=year) file_name = '{0}all_forecasts_{1}_lk{2}.pickle'.format( env.local_storage, year, lang_key) file_date_limit = dt.datetime.now() - dt.timedelta(hours=max_file_age) # if we are well out of the current season (30 days) its little chance the data set has changed. current_season = gm.get_season_from_date(dt.date.today() - dt.timedelta(30)) # Get forecast regions used in the current year region_ids = gm.get_forecast_regions(year, get_b_regions=True) get_new = True if os.path.exists(file_name): # if file contains a season long gone, dont make new. if year == current_season: file_age = dt.datetime.fromtimestamp(os.path.getmtime(file_name)) # If file is newer than the given time limit, dont make new. if file_age > file_date_limit: # If file size larger than that of an nearly empty file, dont make new. if os.path.getsize(file_name) > 100: # 100 bytes limit get_new = False else: get_new = False if get_new: lg.info( "getvarsompickles.py -> get_all_forecasts: Get new {0} forecasts and pickle." .format(year)) all_forecasts = gfa.get_avalanche_warnings(region_ids, from_date, to_date, lang_key=lang_key) # Valid forecasts have a danger level. The other are empty. valid_forecasts = [] for f in all_forecasts: if f.danger_level > 0: valid_forecasts.append(f) mp.pickle_anything(valid_forecasts, file_name) else: valid_forecasts = mp.unpickle_anything(file_name) return valid_forecasts
def make_forecasts_for_Espen_at_sweco(): """Hei. I forbindelse med et prosjekt i Sørreisa i Troms ønsker vi å gi råd til vår kunde om evakuering av bygg i skredutsatt terreng. Som en del av vår vurdering hadde det vært veldig nyttig med statistikk for varslingen, altså statistikk om hvor ofte de ulike faregradene er varslet. Er det mulig å få tak i slik statistikk? Gjerne så langt tilbake i tid som mulig. Vennlig hilsen Espen Eidsvåg""" pickle_file_name = '{0}forecasts_sorreisa_espen.pickle'.format( env.local_storage) get_new = True all_dangers = [] if get_new: years = ['2012-13', '2013-14', '2014-15', '2015-16'] region_ids = [110, 112] # Senja, Bardu for y in years: from_date, to_date = gm.get_forecast_dates(y) for region_id in region_ids: all_dangers += gd.get_forecasted_dangers( region_id, from_date, to_date) years = ['2016-17', '2017-18', '2018-19'] region_ids = [3012, 3013] # Sør Troms, Indre Troms for y in years: from_date, to_date = gm.get_forecast_dates(y) for region_id in region_ids: all_dangers += gd.get_forecasted_dangers( region_id, from_date, to_date) mp.pickle_anything(all_dangers, pickle_file_name) else: all_dangers = mp.unpickle_anything(pickle_file_name) output_forecast_problems = '{0}Varsel for Sørreisa.Espen Eidsvåg Sweco.csv'.format( env.output_folder) import collections as coll # Write forecasts to file with open(output_forecast_problems, 'w', encoding='utf-8') as f: make_header = True for d in all_dangers: for p in d.avalanche_problems: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(p.date, '%Y-%m-%d')), ('Region id', p.region_regobs_id), ('Region', p.region_name), ('DL', p.danger_level), ('Danger level', p.danger_level_name), ('Problem order', p.order), ('Problem', p.problem), ('Cause/ weaklayer', p.cause_name), ('Type', p.aval_type), ('Size', p.aval_size), ('Trigger', p.aval_trigger), ('Probability', p.aval_probability), ('Distribution', p.aval_distribution) ]) if make_header: f.write( ' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n')
def plot_season_for_all_regobs_locations(year='2018-19', calculate_new=False, get_new_obs=False, make_plots=False, delete_old_plots=False): """Method specialized for scheduled plotting for iskart.no. Method makes a season plot for all ObsLocations in regObs where we have a first ice date. It may take some time to plot. 250 lakes for a season and for each plot weather params are requested from the GTS. The workings of the method: 1. get all locations ids and belonging observations where we have first ice. 2.1 if calculate new, empty sesong folder and pickle in local storage and calculate (and make plots if requested). 2.2 Make metadata json for showing files on iskart.no 3. All calculations are compared to observed data in scatter plot. :param year: [String] Season for plotting. eg: '2016-17' :param calculate_new: [bool] Calculate new ice thicks. If false only make the seasonal scatter. :param get_new_obs: [bool] :param make_plots: [bool] If False all calculations are made, but only the scatter comparison against observatiosn is ploted :param delete_old_plots: [bool] If True all former plots and pickles are removed. """ pickle_file_name_and_path = '{0}all_calculated_ice_{1}.pickle'.format(se.local_storage, year) location_id_metadata_json = '{}location_id_metadata.json'.format(se.sesong_plots_folder) if calculate_new: if delete_old_plots: # Empty the sesong plot folder for file in os.listdir(se.sesong_plots_folder): file_path = os.path.join(se.sesong_plots_folder, file) try: if os.path.isfile(file_path): os.unlink(file_path) except OSError: pass # remove pickle old data - because we are getting new try: os.remove(pickle_file_name_and_path) except OSError: pass all_observations = gro.get_all_season_ice(year, get_new=get_new_obs) from_date, to_date = gm.get_dates_from_year(year) all_calculated = {} all_observed = {} location_id_metadata = {} for location_id, observed_ice in all_observations.items(): try: calculated, observed, plot_filename = _plot_season( location_id, from_date, to_date, observed_ice, make_plots=make_plots, plot_folder=se.sesong_plots_folder) all_calculated[location_id] = calculated all_observed[location_id] = observed except: error_msg = sys.exc_info()[0] ml.log_and_print("[error] calculateandplot.py -> plot_season_for_all_regobs_locations: Error making plot for {}".format(error_msg, location_id)) # Make the json with metadata needed for iskart.no. Add only if the plot was made and thus file exists. if os.path.isfile(se.sesong_plots_folder + plot_filename): region_name = observed_ice[0].metadata['OriginalObject']['ForecastRegionName'] if not region_name: region_name = 'Ukjent region' lake_id = observed_ice[0].metadata['LocationID'] x, y = observed_ice[0].metadata['UTMEast'], observed_ice[0].metadata['UTMNorth'] lake_name = observed_ice[0].metadata['LocationName'] if not lake_name: lake_name = 'E{} N{}'.format(x, y) location_id_metadata[location_id] = {'RegionName': region_name, 'LakeID': lake_id, 'LakeName': lake_name, 'PlotFileName': plot_filename} mp.pickle_anything([all_calculated, all_observed], pickle_file_name_and_path) try: json_string = json.dumps(location_id_metadata, ensure_ascii=False).encode('utf-8') with open(location_id_metadata_json, 'wb') as f: f.write(json_string) except: error_msg = sys.exc_info()[0] ml.log_and_print("[error]calculateandplot.py -> plot_season_for_all_regobs_locations: Cant write json. {}".format(error_msg)) else: [all_calculated, all_observed] = mp.unpickle_anything(pickle_file_name_and_path) try: pts.scatter_calculated_vs_observed(all_calculated, all_observed, year) except: error_msg = sys.exc_info()[0] ml.log_and_print("[error] calculateandplot.py -> plot_season_for_all_regobs_locations: {}. Could not plot scatter {}.".format(error_msg, year))
def calculate_and_plot9d_season(period='2018-19'): """Calculate ice columns for 9 days and make plots of all ice thickness for a given season or optionally 'Today'. The inner workings: 1.1 Retrieves ice thickness observations from regObs. If period is given as a season, all observations for this season will be requested. All previous plots and local storage will be deleted. 1.2 If period='Today' ice thickness observations from today will be requested and plotted. Older plots will be in the folder. Metadata dict will be merged. 2. Calculate the 9 day prognosis from the observation time and plots the result. 3. Make a metadata json for handling files on iskart.no. Only confirmed files in folder will be added to metadata json. :param period: [String] Default is current season (2017-18). :return: """ log_referance = 'calculateandplot.py -> calculate_and_plot9d_season' # File names regid_metadata_json = '{}regid_metadata.json'.format(se.ni_dogn_plots_folder) regid_metadata_pickle = '{}regid_metadata.pickle'.format(se.local_storage) if period == 'Today': ice_thicks = gro.get_ice_thickness_today() else: # Empty the 9dogn folder # for file in os.listdir(se.ni_dogn_plots_folder): # file_path = os.path.join(se.ni_dogn_plots_folder, file) # try: # if os.path.isfile(file_path): # os.unlink(file_path) # except OSError: # pass # remove pickle with metadata try: os.remove(regid_metadata_pickle) except OSError: pass # Get new observations ice_thicks = gro.get_ice_thickness_observations(period, reset_and_get_new=True) # Calculate and plot for k, v in ice_thicks.items(): # If file is missing, make it. If observation is older than 11 days it is based on gridded data for sure and no plot file needed. make_plot = False max_file_age = 11 date_limit = dt.datetime.now() - dt.timedelta(days=max_file_age) file_names = os.listdir(se.ni_dogn_plots_folder) plot_filename = '{0}.png'.format(k) if plot_filename not in file_names: make_plot = True else: if v.date.date() > date_limit.date(): make_plot = True if make_plot: try: calculate_and_plot9d_regid(k, plot_folder=se.ni_dogn_plots_folder, observed_ice=v) except: error_msg = sys.exc_info()[0] ml.log_and_print("[Error] {} Error making plot for {} {}".format(log_referance, k, error_msg)) # Make json with metadata for using files on iskart.no. Load metadata from pickle if available and # new observations where a plot is available will be made. if not os.path.exists(regid_metadata_pickle): regid_metadata = {} else: regid_metadata = mp.unpickle_anything(regid_metadata_pickle) list_of_plots = os.listdir(se.ni_dogn_plots_folder) for k, v in ice_thicks.items(): # only add metadata on files that are in the folder if '{0}.png'.format(k) in list_of_plots: date = v.date.date() region_name = v.metadata['OriginalObject']['ForecastRegionName'] if not region_name: region_name = 'Ukjent region' x, y = v.metadata['UTMEast'], v.metadata['UTMNorth'] lake_id = v.metadata['LocationID'] lake_name = v.metadata['LocationName'] if not lake_name: lake_name = 'E{} N{}'.format(x,y) regid_metadata[k] = {'RegionName':region_name,'LakeID':lake_id,'LakeName':lake_name,'Date':'{}'.format(date)} mp.pickle_anything(regid_metadata, regid_metadata_pickle) json_string = json.dumps(regid_metadata, ensure_ascii=False).encode('utf-8') with open(regid_metadata_json, 'wb') as f: f.write(json_string)
def make_forecasts_for_Thea(): """July 2018: Make list of avalanche forecasts danger levels for regions Voss, Romsdalen, Svartisen and Salten (and those before them) for Thea Møllerhaug Lunde (Jernbanedirektoratet). Voss-Bergen ligger i for det meste i Voss-regionen vår. Mo i Rana-Fauske ligger i Svartisen og Salten. Åndalsnes-Bjorli ligger i varslingsregionen Romsdalen.""" pickle_file_name = '{0}201807_avalanche_forecasts_thea.pickle'.format( env.local_storage) get_new = False all_dangers = [] if get_new: # Get Voss. ForecastRegionTID 124 form 2012-2016 and 3031 since. # Get Romsdalen. ForecastRegionTID 118 from 2012-2016 and 3023 since. # Get Svartisen. ForecastRegionTID 131 from 2012-2016 and 3017 since. # Get Salten. ForecastRegionTID 133 form 2012-2016 and 3016 since. years = ['2012-13', '2013-14', '2014-15', '2015-16'] region_ids = [124, 118, 131, 133] for y in years: from_date, to_date = gm.get_forecast_dates(y) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) years = ['2016-17', '2017-18'] region_ids = [3031, 3023, 3017, 3016] for y in years: from_date, to_date = gm.get_forecast_dates(y) all_dangers += gd.get_forecasted_dangers(region_ids, from_date, to_date) mp.pickle_anything(all_dangers, pickle_file_name) else: all_dangers = mp.unpickle_anything(pickle_file_name) output_forecast_problems = '{0}201807 Snøskredvarsel for Thea.txt'.format( env.output_folder) import collections as coll # Write forecasts to file with open(output_forecast_problems, 'w', encoding='utf-8') as f: make_header = True for d in all_dangers: out_data = coll.OrderedDict([ ('Date', dt.date.strftime(d.date, '%Y-%m-%d')), ('Region id', d.region_regobs_id), ('Region', d.region_name), ('DL', d.danger_level), ('Danger level', d.danger_level_name), ]) if make_header: f.write(' ;'.join([fe.make_str(d) for d in out_data.keys()]) + '\n') make_header = False f.write(' ;'.join([fe.make_str(d) for d in out_data.values()]) + '\n') pass
def get_kdv(view): """Imports a view view from regObs and returns a dictionary with <key, value> = <ID, Name> An view is requested from the regObs api if the pickle file is older than 3 days. :param view: [string] kdv view :return dict: {} view as a dictionary Ex of use: aval_cause_kdv = get_kdv('AvalCauseKDV') Ex of url for returning values for IceCoverKDV in norwegian: http://api.nve.no/hydrology/regobs/v0.9.4/OData.svc/ForecastRegionKDV?$filter=Langkey%20eq%201%20&$format=json """ kdv_file_name = '{0}{1}.pickle'.format(env.local_storage, view) dict = {} if os.path.exists(kdv_file_name): max_file_age = 3 # file_date_seconds = os.path.getctime(kdv_file_name) file_date_seconds = os.path.getmtime(kdv_file_name) file_date_datetime = dt.datetime.fromtimestamp(file_date_seconds) file_date_limit = dt.datetime.now() - dt.timedelta(days=max_file_age) if file_date_datetime < file_date_limit: ml.log_and_print("[info] getkdvelements.py -> get_kdv: Old xKDV. Removing file from local storage: {0}".format(kdv_file_name)) os.remove(kdv_file_name) ordered_dict = get_kdv(view) mp.pickle_anything(ordered_dict, kdv_file_name) else: # ml.log_and_print("[info] getkdvelements.py -> get_kdv: Getting KDV from local storage: {0}".format(kdv_file_name)) ordered_dict = mp.unpickle_anything(kdv_file_name, print_message=False) else: filter = 'filter=Langkey%20eq%201' if 'TripTypeKDV' in view: filter = 'filter=LangKey%20eq%201' url = 'https://api.nve.no/hydrology/regobs/{0}/OData.svc/{1}?${2}&$format=json'.format(env.odata_version, view, filter) lang_key = 1 print("getkdvelements.py -> get_kdv: Getting KDV from URL: {0}".format(url)) kdv = requests.get(url).json() for a in kdv['d']['results']: try: sort_order = a['SortOrder'] is_active = a['IsActive'] if 'AvalCauseKDV' in url and 9 < int(a['ID']) < 26: # this table gets special treatment. Short names are in description and long names are in Name. id = int(a['ID']) name = a['Description'] description = a['Name'] elif 'TripTypeKDV' in view: id = int(a['TripTypeTID']) name = a['Name'] description = a['Descr'] else: id = int(a['ID']) name = a['Name'] description = a['Description'] dict[id] = vc.KDVelement(id, sort_order, is_active, name, description, lang_key) except (RuntimeError, TypeError, NameError): pass ordered_dict = collections.OrderedDict(sorted(dict.items())) mp.pickle_anything(ordered_dict, kdv_file_name) return ordered_dict
def get_ice_thickness_observations(year, reset_and_get_new=False): """Gets all the observed ice thickness (RegistrationTID = 50) from regObs for one year. The inner workings of the method: 1. We have an option of resetting local storage (delete pickle) and thus forcing the get_new. 2.1 Try opening a pickle, if it doesnt exist, an exception is thrown and we get new data. 2.2 If the requested data is from a previous season, no changes are expected, so load the pickle without adding the last observations registered in regObs. Anyway, don't get new data. 2.3 If the requested data is from this season, set request from_date to the last modified date of the pickle and 7 days past that. Add these last obs to the pickle data, and thus it is not necessary to get new. 3. If get new, it gets all new data for the season. 4. Else, load pickle and if some last obs are to be added, do so. :param year: [string] Eg '2017-18' :param reset_and_get_new: [bool] :return: ice_thickeness_obs_dict """ log_referance = 'getregobsdata.py -> get_ice_thickness_observations' pickle_file_name = '{0}get_ice_thickness_observations_{1}.pickle'.format(se.local_storage, year) # 1. Remove pickle if it exists, forcing the get_new if reset_and_get_new: try: os.remove(pickle_file_name) except OSError: pass from_date, to_date = gm.get_dates_from_year(year) add_last_obs = None get_new = None try: mtime = os.path.getmtime(pickle_file_name) last_modified_date = dt.datetime.fromtimestamp(mtime).date() # if file newer than the season (that is, if this is historical data), load it without requesting new. dt_to_date = dt.datetime.strptime(to_date, '%Y-%m-%d').date() if last_modified_date > dt_to_date: add_last_obs = False else: add_last_obs = True to_date = dt.date.today() from_date = last_modified_date - dt.timedelta(days=7) get_new = False except OSError: # file does not exists, so get_new. ml.log_and_print("{0}: No matching pickle found, getting new data.".format(log_referance)) get_new = True if get_new: ml.log_and_print('{0}: Getting new for year {1}.'.format(log_referance, year)) ice_thickeness_obs = get_data(from_date=from_date, to_date=to_date, registration_types=50, geohazard_tids=70) ice_thickeness_obs_dict = {} for o in ice_thickeness_obs: if o['RegistrationTid'] == 50: ice_column = _webapi_ice_col_to_ice_class(o) if ice_column is not None: ice_thickeness_obs_dict[o['RegId']] = ice_column mp.pickle_anything(ice_thickeness_obs_dict, pickle_file_name) else: ice_thickeness_obs_dict = mp.unpickle_anything(pickle_file_name) if add_last_obs: ml.log_and_print("{0}: Adding observations from {1} to {2}".format(log_referance, from_date, to_date)) new_ice_thickeness_obs = get_data(from_date=from_date, to_date=to_date, registration_types=50, geohazard_tids=70) new_ice_thickeness_obs_dict = {} for o in new_ice_thickeness_obs: if o['RegistrationTid'] == 50: ice_column = _webapi_ice_col_to_ice_class(o) if ice_column is not None: new_ice_thickeness_obs_dict[o['RegId']] = ice_column for k,v in new_ice_thickeness_obs_dict.items(): ice_thickeness_obs_dict[k] = v mp.pickle_anything(ice_thickeness_obs_dict, pickle_file_name) return ice_thickeness_obs_dict
def add_Inflow_DOP(self, Inflow_DOP_inn): messages = we.test_for_missing_elements(Inflow_DOP_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOP = Inflow_DOP_inn def add_Inflow_Chla(self, Inflow_Chla_inn): messages = we.test_for_missing_elements(Inflow_Chla_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_Chla = Inflow_Chla_inn def add_Inflow_DOC(self, Inflow_DOC_inn): messages = we.test_for_missing_elements(Inflow_DOC_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOC = Inflow_DOC_inn if __name__ == "__main__": yesturday = (dt.date.today()-dt.timedelta(days=1)).strftime("%Y-%m-%d") #harvest_and_save_blindern('2000-01-01', yesturday) #harvest_and_save_nordnesfjelet('2014-08-01', yesturday) data = harvest_for_mylake_hakkloa('2013-04-01', '2015-10-01') mp.pickle_anything(data, data.output_file_path +'.pickle') data2 = mp.unpickle_anything('{0}HAK_input'.format(env.data_path) +'.pickle') mfd.write_mylake_inputfile(data2)