def get_incident(from_date, to_date, region_ids=None, observer_ids=None, output='List', geohazard_tid=None): """Gets observations from IncidentV. View is shared by all the geohazards so the filter includes geohazard_tid if only one geohazard is needed. :param from_date: [date] A query returns [from_date, to_date> :param to_date: [date] A query returns [from_date, to_date> :param region_ids: [int or list of ints] If region_ids = None, all regions are selected :param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected :param output: [string] Options: 'List', 'DataFrame' and 'Count'. Default 'List'. :param geohazard_tid [int] 10 is snow, 20,30,40 are dirt, 60 is water and 70 is ice :return: """ if output == 'List' or output == 'DataFrame': data = _make_data_request("IncidentV", from_date, to_date, region_ids, observer_ids, geohazard_tid) list = [Incident(d) for d in data] list = sorted(list, key=lambda DangerObs: DangerObs.DtObsTime) if output == 'List': return list elif output == 'DataFrame': return _make_data_frame(list) elif output == 'Count': count = _make_count_request("IncidentV", from_date, to_date, region_ids, observer_ids, geohazard_tid) return count else: ml.log_and_print('getobservations.py -> get_incident: Illegal output option.') return None
def get_avalanche(from_date, to_date, region_ids=None, observer_ids=None, output='List'): """Gets observations from AvalancheObsV. :param from_date: [date] A query returns [from_date, to_date> :param to_date: [date] A query returns [from_date, to_date> :param region_ids: [int or list of ints] If region_ids = None, all regions are selected :param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected :param output: [string] Options: 'List', 'DataFrame' and 'Count'. Default 'List'. :return: """ if output == 'List' or output == 'DataFrame': data = _make_data_request("AvalancheObsV", from_date, to_date, region_ids, observer_ids) list = [AvalancheObs(d) for d in data] list = sorted(list, key=lambda AvalancheObs: AvalancheObs.DtObsTime) if output == 'List': return list elif output == 'DataFrame': return _make_data_frame(list) elif output == 'Count': count = _make_count_request("AvalancheObsV", from_date, to_date, region_ids, observer_ids) return count else: ml.log_and_print('getobservations.py -> get_avalanche: Illegal output option.') return None
def _view_test(): '''Method tests if views show observartions made by Ragnar on a test location the last fiew days. The purpose is to test if views correctly remove deleted items. :return: ''' views = ['AllRegistrationsV', 'AllRegistrationsIceV', 'AllRegistrationsLandslideV', 'AllRegistrationsSnowV', 'AllRegistrationsWaterV', 'DangerObsV', 'IncidentV', 'GeneralObservationV', 'PictureV', 'AvalancheObsV', 'AvalancheActivityObsV', # 'AvalancheActivityObs2V', 'AvalancheDangerObsV', 'AvalancheEvaluationV', 'AvalancheEvaluation2V', 'AvalancheEvaluation3V', 'AvalancheEvalProblem2V', 'WeatherObservationV', 'ColumnTestV', # 'DangerSignV', # 'WeakLayerV', 'SnowCoverObsV', 'SnowSurfaceObservationV', 'IceCoverObsV', 'IceThicknessV', 'LandSlideObsV', 'WaterLevelV', ] log = env.log view_test = [] for v in views: view_test += _make_data_request(v, 0, dt.date.today() - dt.timedelta(days=1), dt.date.today() + dt.timedelta(days=1)) for o in view_test: if 'Ragnar' in o['NickName'] and 'Test' in o['LocationName']: ml.log_and_print('{0: <30} with RegID {3: <7} has observations by {1: <20} on location {2}.'.format(o['__metadata']['type'], o['NickName'], o['LocationName'], o['RegID']), print_it=True, log_it=True) a = 1
def _make_count_request(view, from_date, to_date, region_ids=None, observer_ids=None, geohazard_tid=None): """Common part of all count requests. Note that some views are shared by all geohazards. In these it is also possible to specify geohazard_tid in the filter. :param view: :param from_date: [date] A query returns [from_date, to_date> :param to_date: [date] A query returns [from_date, to_date> :param region_ids: [int or list of ints] If region_ids = None, all regions are selected :param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected :param geohazard_tid: [int] 10 is snow, 20,30,40 are dirt, 60 is water and 70 is ice :return: [int] Number of observations. """ # If input isn't a list, make it so if not isinstance(region_ids, types.ListType): region_ids = [region_ids] if not isinstance(observer_ids, types.ListType): observer_ids = [observer_ids] count = 0 for region_id in region_ids: for observer_id in observer_ids: odata_query = _make_odata_filter(from_date, to_date, region_id, observer_id, geohazard_tid) url = "http://api.nve.no/hydrology/regobs/{0}/Odata.svc/{1}/$count/?$filter={2}&$format=json"\ .decode('utf8').format(env.api_version, view, odata_query) ml.log_and_print("getobservations.py -> _make_count_request: ..to {0}".format(fe.remove_norwegian_letters(url))) result = requests.get(url).json() count += result return count
def get_warnings_as_json(region_ids, start_date, end_date, lang_key=1, recursive_count=5): """Selects warnings and returns the json structured result as given on the api. :param region_id: [int or list of ints] RegionID as given in the forecast api [1-99] or in regObs [101-199] :param start_date: [date or string as yyyy-mm-dd] :param end_date: [date or string as yyyy-mm-dd] :param recursive_count [int] by default atempt the same request # times before giving up :return warnings: [string] String as json Eg. http://api01.nve.no/hydrology/forecast/avalanche/v2.0.2/api/AvalancheWarningByRegion/Detail/10/1/2013-01-10/2013-01-20 """ # If input isn't a list, make it so if not isinstance(region_ids, types.ListType): region_ids = [region_ids] warnings = [] recursive_count_default = recursive_count # need the default for later for region_id in region_ids: if len(region_ids) > 1: # if we are looping the initial list make sure each item gets the recursive count default recursive_count = recursive_count_default if region_id > 100: region_id = region_id - 100 md.log_and_print("getForecastApi -> get_warnings_as_json: Getting AvalancheWarnings for {0} from {1} til {2}"\ .format(region_id, start_date, end_date)) url = "http://api01.nve.no/hydrology/forecast/avalanche/v2.0.2/api/AvalancheWarningByRegion/Detail/{0}/{3}/{1}/{2}"\ .format(region_id, start_date, end_date, lang_key) # If at first you don't succeed, try and try again. try: warnings += requests.get(url).json() md.log_and_print("getForecastApi -> get_warnings_as_json: {0} warnings found for {1}.".format(len(warnings), region_id)) except: md.log_and_print("getForecastApi -> get_warnings_as_json: EXCEPTION. RECURSIVE COUNT {0}".format(recursive_count)) if recursive_count > 1: recursive_count -= 1 # count down warnings += get_warnings_as_json(region_id, start_date, end_date, lang_key, recursive_count=recursive_count) return warnings
def _make_data_request(view, from_date, to_date, region_ids=None, observer_ids=None, geohazard_tid=None, recursive_count=5): """Common part of all data requests. :param view: [string] Name of view in regObs :param from_date: [date] A query returns [from_date, to_date> :param to_date: [date] A query returns [from_date, to_date> :param region_ids: [int or list of ints] If region_ids = None, all regions are selected :param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected :param geohazard_tid: [int] 10 is snow, 20,30,40 are dirt, 60 is water and 70 is ice :param recursive_count [int] by default atempt the same request 5 times before giving up :return: untreated request result """ # If input isn't a list, make it so if not isinstance(region_ids, types.ListType): region_ids = [region_ids] if not isinstance(observer_ids, types.ListType): observer_ids = [observer_ids] data_out = [] recursive_count_default = recursive_count # need the default for later for region_id in region_ids: for observer_id in observer_ids: if len(observer_ids) > 1: # if we are looping the initial list make sure each item gets the recursive count default recursive_count = recursive_count_default odata_query = _make_odata_filter(from_date, to_date, region_id, observer_id, geohazard_tid) url = "http://api.nve.no/hydrology/regobs/{0}/Odata.svc/{1}/?$filter={2}&$format=json"\ .decode('utf8').format(env.api_version, view, odata_query) if view == "IncidentV": url = url.replace('GeoHazardTID', 'GeoHazardTid') ml.log_and_print("getobservations.py -> _make_data_request: {0}".format(fe.remove_norwegian_letters(url))) try: result = requests.get(url).json() result = result['d']['results'] except: # Need to improve on exception handling: http://docs.python-requests.org/en/latest/user/quickstart/#errors-and-exceptions ml.log_and_print("getobservations.py -> _make_data_request: EXCEPTION. RECURSIVE COUNT {0}".format(recursive_count)) result = [] if recursive_count > 1: recursive_count -= 1 # count down result = _make_data_request(view, from_date, to_date, region_id, observer_id, geohazard_tid, recursive_count=recursive_count) # if more than 1000 elements are requested, odata truncates data to 1000. We do more requests if len(result) == 1000: time_delta = to_date - from_date date_in_middle = from_date + time_delta/2 result = _make_data_request(view, from_date, date_in_middle, region_id, observer_id, geohazard_tid) \ + _make_data_request(view, date_in_middle, to_date, region_id, observer_id, geohazard_tid) data_out += result env.log.append("{2} to {3}: {1} has {0} items".format(len(data_out), view, from_date, to_date)) return data_out
# all_registrations_snow = get_all_registrations(from_date, to_date, geohazard_tid=10) # all_registrations_ice = get_all_registrations(from_date, to_date, geohazard_tid=70) # all_registrations = get_all_registrations(from_date, to_date) # # count_registrations_snow = get_all_registrations(from_date, to_date, geohazard_tid=10, output='Count') # count_registrations_ice = get_all_registrations(from_date, to_date, geohazard_tid=70, output='Count') # count_registrations = get_all_registrations(from_date, to_date, output='Count') # avalanche_activity = get_avalanche_activity(from_date, to_date, 116) avalanche_activity_2 = get_avalanche_activity_2(from_date, to_date, 116) # avalanche = get_avalanche(from_date, to_date, region_ids) # danger_sign = get_danger_sign(from_date, to_date, observer_ids) count_registrations_snow_10 = get_all_registrations(dt.date(2015, 12, 1), dt.date.today(), geohazard_tid=10, output='Count', observer_ids=10) # _view_test() all_registrations = get_all_registrations(from_date, to_date) for a in all_registrations: ml.log_and_print( "{0}\t{1}\t{2:<6}\t{3:<15}\t{4:<20}\t{5:<20}\t{6}".format(a.RegID, a.DtRegTime, a.GeoHazardName, a.MunicipalName, a.ForecastRegionName, a.NickName, a.RegistrationName), print_it=True, log_it=True) a = 1