def pickle_warnings(regions, date_from, date_to, pickle_file_name): ''' All warnings and problems are selected from regObs or the avalanche api and neatly pickel'd for later use. :param regions: list [] :param date_from: string as 'yyyy-mm-dd' :param date_to: string as 'yyyy-mm-dd' :param pickle_file_name: filename including directory as string :return: ''' warnings = [] for r in regions: # get all warning and problems for this region and then loop though them joining them on date __warnings = GFA.get_warnings(r, date_from, date_to) __name = GRO.get_forecast_region_name(r) __problems = GRO.get_problems_from_AvalancheWarnProblemV(__name, r, date_from, date_to) print '{0} problems found for {1}'.format(len(__problems), __name) for i in range(0, len(__warnings), 1): for j in range(0, len(__problems), 1): if __warnings[i].date == __problems[j].date: __warnings[i].add_problem(__problems[j]) warnings = warnings + __warnings RP.pickle_anything(warnings, pickle_file_name)
def pickle_data_set(warnings, file_name, use_ikke_gitt): ''' Takes the warnings data set which is a list of class AvalancheDanger objects and makes a nested dictionary data set of if. This makes the data set easier to distribute and use if the AvalancheDanger and AvalancheProblem classes are missing. The data set also includes information on what the xKDV tables in regObs contains and preferred colors when plotting. :param warnings: list of AvalancheDanger objects :param file_name: filename to pickle the data to :param use_ikke_gitt: If I dont whant to use the ID = 0 (Ikke gitt) values they can be omitted all in all. :return: ''' level_list = [] size_list = [] trigger_list = [] probability_list = [] distribution_list = [] for w in warnings: if w.danger_level > 0 and len(w.avalanche_problems) > 0: level_list.append(w.danger_level) size_list.append(w.avalanche_problems[0].aval_size) trigger_list.append(w.avalanche_problems[0].aval_trigger) probability_list.append(w.avalanche_problems[0].aval_probability) distribution_list.append(w.avalanche_problems[0].aval_distribution) level_keys = GRO.get_kdv('AvalancheDangerKDV').keys() size_keys = GRO.get_kdv('DestructiveSizeKDV').values() triggers_keys = GRO.get_kdv('AvalTriggerSimpleKDV').values() probability_keys = GRO.get_kdv('AvalProbabilityKDV').values() distribution_keys = GRO.get_kdv('AvalPropagationKDV').values() level_colors = ['0.5','#ccff66', '#ffff00', '#ff9900', '#ff0000', 'k'] if use_ikke_gitt == False: level_keys.pop(0) size_keys.pop(0) triggers_keys.pop(0) probability_keys.pop(0) distribution_keys.pop(0) level_colors.pop(0) data_set = {'level': {'values': level_list, 'keys': level_keys, 'colors':level_colors}, 'size': {'values': size_list, 'keys': size_keys, 'colors':['0.7']}, 'trigger': {'values': trigger_list, 'keys': triggers_keys, 'colors':['0.7']}, 'probability': {'values': probability_list, 'keys': probability_keys, 'colors':['0.7']}, 'distribution': {'values': distribution_list, 'keys': distribution_keys, 'colors':['0.7']}} RP.pickle_anything(data_set, file_name)
def get_kdv(x_kdv): ''' Imports a x_kdv view from regObs and returns a dictionary with <key, value> = <ID, Name> An x_kdv is requested from the regObs api if a pickle file newer than a week exists. :param x_kdv: [string] x_kdv view :return dict: {} x_kdv as a dictionary Ex of use: aval_cause_kdv = get_kdv('AvalCauseKDV') Ex of url for returning values for IceCoverKDV in norwegian: http://api.nve.no/hydrology/regobs/v0.9.4/OData.svc/ForecastRegionKDV?$filter=Langkey%20eq%201%20&$format=json ''' kdv_file = '{0}{1}.pickle'.format(kdv_elements_folder, x_kdv) dict = {} if os.path.exists(kdv_file): ### Should be useful to test if the file is old and if so make a new one # max_file_age = 3 # file_date = time.ctime(os.path.getctime(kdv_file)) # date_limit = datetime.datetime.now() - datetime.timedelta(days=max_file_age) ### #print("Getting KDV from pickle:{0}".format(kdv_file)) dict = mp.unpickle_anything(kdv_file) else: url = 'http://api.nve.no/hydrology/regobs/{0}/OData.svc/{1}?$filter=Langkey%20eq%201%20&$format=json'\ .format(api_version, x_kdv) print("Getting KDV from URL:{0}".format(url)) kdv = requests.get(url).json() for a in kdv['d']['results']: try: if 'AvalCauseKDV' in url and a['ID'] > 9 and a['ID'] < 26: # this table gets special treatment dict[a["ID"]] = fe.remove_norwegian_letters(a["Description"]) else: dict[a["ID"]] = fe.remove_norwegian_letters(a["Name"]) except (RuntimeError, TypeError, NameError): pass mp.pickle_anything(dict, kdv_file) return dict
def add_Inflow_DOP(self, Inflow_DOP_inn): messages = we.test_for_missing_elements(Inflow_DOP_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOP = Inflow_DOP_inn def add_Inflow_Chla(self, Inflow_Chla_inn): messages = we.test_for_missing_elements(Inflow_Chla_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_Chla = Inflow_Chla_inn def add_Inflow_DOC(self, Inflow_DOC_inn): messages = we.test_for_missing_elements(Inflow_DOC_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOC = Inflow_DOC_inn if __name__ == "__main__": yesturday = (dt.date.today()-dt.timedelta(days=1)).strftime("%Y-%m-%d") #harvest_and_save_blindern('2000-01-01', yesturday) #harvest_and_save_nordnesfjelet('2014-08-01', yesturday) data = harvest_for_mylake_hakkloa('2013-04-01', '2015-10-01') mp.pickle_anything(data, data.output_file_path +'.pickle') data2 = mp.unpickle_anything('{0}HAK_input'.format(env.data_path) +'.pickle') mfd.write_mylake_inputfile(data2)
def add_Inflow_DOP(self, Inflow_DOP_inn): messages = we.test_for_missing_elements(Inflow_DOP_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOP = Inflow_DOP_inn def add_Inflow_Chla(self, Inflow_Chla_inn): messages = we.test_for_missing_elements(Inflow_Chla_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_Chla = Inflow_Chla_inn def add_Inflow_DOC(self, Inflow_DOC_inn): messages = we.test_for_missing_elements(Inflow_DOC_inn, self.from_date, self.to_date) self.metadata += messages self.Inflow_DOC = Inflow_DOC_inn if __name__ == "__main__": yesturday = (dt.date.today() - dt.timedelta(days=1)).strftime("%Y-%m-%d") #harvest_and_save_blindern('2000-01-01', yesturday) #harvest_and_save_nordnesfjelet('2014-08-01', yesturday) data = harvest_for_mylake_hakkloa('2013-04-01', '2015-10-01') mp.pickle_anything(data, data.output_file_path + '.pickle') data2 = mp.unpickle_anything('{0}HAK_input'.format(env.data_path) + '.pickle') mfd.write_mylake_inputfile(data2)