def get_incident(from_date, to_date, region_ids=None, observer_ids=None, output='List', geohazard_tid=None):
"""Gets observations from IncidentV. View is shared by all the geohazards so the filter includes
geohazard_tid if only one geohazard is needed.
:param from_date: [date] A query returns [from_date, to_date>
:param to_date: [date] A query returns [from_date, to_date>
:param region_ids: [int or list of ints] If region_ids = None, all regions are selected
:param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected
:param output: [string] Options: 'List', 'DataFrame' and 'Count'. Default 'List'.
:param geohazard_tid [int] 10 is snow, 20,30,40 are dirt, 60 is water and 70 is ice
:return:
"""
if output == 'List' or output == 'DataFrame':
data = _make_data_request("IncidentV", from_date, to_date, region_ids, observer_ids, geohazard_tid)
list = [Incident(d) for d in data]
list = sorted(list, key=lambda DangerObs: DangerObs.DtObsTime)
if output == 'List':
return list
elif output == 'DataFrame':
return _make_data_frame(list)
elif output == 'Count':
count = _make_count_request("IncidentV", from_date, to_date, region_ids, observer_ids, geohazard_tid)
return count
else:
ml.log_and_print('getobservations.py -> get_incident: Illegal output option.')
return None
def get_avalanche(from_date, to_date, region_ids=None, observer_ids=None, output='List'):
"""Gets observations from AvalancheObsV.
:param from_date: [date] A query returns [from_date, to_date>
:param to_date: [date] A query returns [from_date, to_date>
:param region_ids: [int or list of ints] If region_ids = None, all regions are selected
:param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected
:param output: [string] Options: 'List', 'DataFrame' and 'Count'. Default 'List'.
:return:
"""
if output == 'List' or output == 'DataFrame':
data = _make_data_request("AvalancheObsV", from_date, to_date, region_ids, observer_ids)
list = [AvalancheObs(d) for d in data]
list = sorted(list, key=lambda AvalancheObs: AvalancheObs.DtObsTime)
if output == 'List':
return list
elif output == 'DataFrame':
return _make_data_frame(list)
elif output == 'Count':
count = _make_count_request("AvalancheObsV", from_date, to_date, region_ids, observer_ids)
return count
else:
ml.log_and_print('getobservations.py -> get_avalanche: Illegal output option.')
return None
def _view_test():
'''Method tests if views show observartions made by Ragnar on a test location the last fiew days.
The purpose is to test if views correctly remove deleted items.
:return:
'''
views = ['AllRegistrationsV',
'AllRegistrationsIceV',
'AllRegistrationsLandslideV',
'AllRegistrationsSnowV',
'AllRegistrationsWaterV',
'DangerObsV',
'IncidentV',
'GeneralObservationV',
'PictureV',
'AvalancheObsV',
'AvalancheActivityObsV',
# 'AvalancheActivityObs2V',
'AvalancheDangerObsV',
'AvalancheEvaluationV',
'AvalancheEvaluation2V',
'AvalancheEvaluation3V',
'AvalancheEvalProblem2V',
'WeatherObservationV',
'ColumnTestV',
# 'DangerSignV',
# 'WeakLayerV',
'SnowCoverObsV',
'SnowSurfaceObservationV',
'IceCoverObsV',
'IceThicknessV',
'LandSlideObsV',
'WaterLevelV',
]
log = env.log
view_test = []
for v in views:
view_test += _make_data_request(v, 0, dt.date.today() - dt.timedelta(days=1), dt.date.today() + dt.timedelta(days=1))
for o in view_test:
if 'Ragnar' in o['NickName'] and 'Test' in o['LocationName']:
ml.log_and_print('{0: <30} with RegID {3: <7} has observations by {1: <20} on location {2}.'.format(o['__metadata']['type'], o['NickName'], o['LocationName'], o['RegID']),
print_it=True, log_it=True)
a = 1
def _make_count_request(view, from_date, to_date, region_ids=None, observer_ids=None, geohazard_tid=None):
"""Common part of all count requests.
Note that some views are shared by all geohazards. In these it is also
possible to specify geohazard_tid in the filter.
:param view:
:param from_date: [date] A query returns [from_date, to_date>
:param to_date: [date] A query returns [from_date, to_date>
:param region_ids: [int or list of ints] If region_ids = None, all regions are selected
:param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected
:param geohazard_tid: [int] 10 is snow, 20,30,40 are dirt, 60 is water and 70 is ice
:return: [int] Number of observations.
"""
# If input isn't a list, make it so
if not isinstance(region_ids, types.ListType):
region_ids = [region_ids]
if not isinstance(observer_ids, types.ListType):
observer_ids = [observer_ids]
count = 0
for region_id in region_ids:
for observer_id in observer_ids:
odata_query = _make_odata_filter(from_date, to_date, region_id, observer_id, geohazard_tid)
url = "http://api.nve.no/hydrology/regobs/{0}/Odata.svc/{1}/$count/?$filter={2}&$format=json"\
.decode('utf8').format(env.api_version, view, odata_query)
ml.log_and_print("getobservations.py -> _make_count_request: ..to {0}".format(fe.remove_norwegian_letters(url)))
result = requests.get(url).json()
count += result
return count
def get_warnings_as_json(region_ids, start_date, end_date, lang_key=1, recursive_count=5):
"""Selects warnings and returns the json structured result as given on the api.
:param region_id: [int or list of ints] RegionID as given in the forecast api [1-99] or in regObs [101-199]
:param start_date: [date or string as yyyy-mm-dd]
:param end_date: [date or string as yyyy-mm-dd]
:param recursive_count [int] by default atempt the same request # times before giving up
:return warnings: [string] String as json
Eg. http://api01.nve.no/hydrology/forecast/avalanche/v2.0.2/api/AvalancheWarningByRegion/Detail/10/1/2013-01-10/2013-01-20
"""
# If input isn't a list, make it so
if not isinstance(region_ids, types.ListType):
region_ids = [region_ids]
warnings = []
recursive_count_default = recursive_count # need the default for later
for region_id in region_ids:
if len(region_ids) > 1:
# if we are looping the initial list make sure each item gets the recursive count default
recursive_count = recursive_count_default
if region_id > 100:
region_id = region_id - 100
md.log_and_print("getForecastApi -> get_warnings_as_json: Getting AvalancheWarnings for {0} from {1} til {2}"\
.format(region_id, start_date, end_date))
url = "http://api01.nve.no/hydrology/forecast/avalanche/v2.0.2/api/AvalancheWarningByRegion/Detail/{0}/{3}/{1}/{2}"\
.format(region_id, start_date, end_date, lang_key)
# If at first you don't succeed, try and try again.
try:
warnings += requests.get(url).json()
md.log_and_print("getForecastApi -> get_warnings_as_json: {0} warnings found for {1}.".format(len(warnings), region_id))
except:
md.log_and_print("getForecastApi -> get_warnings_as_json: EXCEPTION. RECURSIVE COUNT {0}".format(recursive_count))
if recursive_count > 1:
recursive_count -= 1 # count down
warnings += get_warnings_as_json(region_id, start_date, end_date, lang_key, recursive_count=recursive_count)
return warnings
def _make_data_request(view, from_date, to_date, region_ids=None, observer_ids=None, geohazard_tid=None, recursive_count=5):
"""Common part of all data requests.
:param view: [string] Name of view in regObs
:param from_date: [date] A query returns [from_date, to_date>
:param to_date: [date] A query returns [from_date, to_date>
:param region_ids: [int or list of ints] If region_ids = None, all regions are selected
:param observer_ids: [int or list of ints] If observer_ids = None, all observers are selected
:param geohazard_tid: [int] 10 is snow, 20,30,40 are dirt, 60 is water and 70 is ice
:param recursive_count [int] by default atempt the same request 5 times before giving up
:return: untreated request result
"""
# If input isn't a list, make it so
if not isinstance(region_ids, types.ListType):
region_ids = [region_ids]
if not isinstance(observer_ids, types.ListType):
observer_ids = [observer_ids]
data_out = []
recursive_count_default = recursive_count # need the default for later
for region_id in region_ids:
for observer_id in observer_ids:
if len(observer_ids) > 1:
# if we are looping the initial list make sure each item gets the recursive count default
recursive_count = recursive_count_default
odata_query = _make_odata_filter(from_date, to_date, region_id, observer_id, geohazard_tid)
url = "http://api.nve.no/hydrology/regobs/{0}/Odata.svc/{1}/?$filter={2}&$format=json"\
.decode('utf8').format(env.api_version, view, odata_query)
if view == "IncidentV":
url = url.replace('GeoHazardTID', 'GeoHazardTid')
ml.log_and_print("getobservations.py -> _make_data_request: {0}".format(fe.remove_norwegian_letters(url)))
try:
result = requests.get(url).json()
result = result['d']['results']
except:
# Need to improve on exception handling: http://docs.python-requests.org/en/latest/user/quickstart/#errors-and-exceptions
ml.log_and_print("getobservations.py -> _make_data_request: EXCEPTION. RECURSIVE COUNT {0}".format(recursive_count))
result = []
if recursive_count > 1:
recursive_count -= 1 # count down
result = _make_data_request(view, from_date, to_date, region_id, observer_id, geohazard_tid, recursive_count=recursive_count)
# if more than 1000 elements are requested, odata truncates data to 1000. We do more requests
if len(result) == 1000:
time_delta = to_date - from_date
date_in_middle = from_date + time_delta/2
result = _make_data_request(view, from_date, date_in_middle, region_id, observer_id, geohazard_tid) \
+ _make_data_request(view, date_in_middle, to_date, region_id, observer_id, geohazard_tid)
data_out += result
env.log.append("{2} to {3}: {1} has {0} items".format(len(data_out), view, from_date, to_date))
return data_out
# all_registrations_snow = get_all_registrations(from_date, to_date, geohazard_tid=10)
# all_registrations_ice = get_all_registrations(from_date, to_date, geohazard_tid=70)
# all_registrations = get_all_registrations(from_date, to_date)
#
# count_registrations_snow = get_all_registrations(from_date, to_date, geohazard_tid=10, output='Count')
# count_registrations_ice = get_all_registrations(from_date, to_date, geohazard_tid=70, output='Count')
# count_registrations = get_all_registrations(from_date, to_date, output='Count')
#
avalanche_activity = get_avalanche_activity(from_date, to_date, 116)
avalanche_activity_2 = get_avalanche_activity_2(from_date, to_date, 116)
# avalanche = get_avalanche(from_date, to_date, region_ids)
# danger_sign = get_danger_sign(from_date, to_date, observer_ids)
count_registrations_snow_10 = get_all_registrations(dt.date(2015, 12, 1), dt.date.today(), geohazard_tid=10, output='Count', observer_ids=10)
# _view_test()
all_registrations = get_all_registrations(from_date, to_date)
for a in all_registrations:
ml.log_and_print(
"{0}\t{1}\t{2:<6}\t{3:<15}\t{4:<20}\t{5:<20}\t{6}".format(a.RegID, a.DtRegTime, a.GeoHazardName, a.MunicipalName,
a.ForecastRegionName, a.NickName, a.RegistrationName),
print_it=True, log_it=True)
a = 1
