def get_sodsum_data(form_input):
'''
Data acquisition for sodsum
Keyword arguments:
form_input -- parameter file for data request obtained from user of WRCC SOD pages
'''
if 'variable' not in form_input.keys() or 'station_ids' not in form_input.keys():
print 'variable and station_id options required!'
sys.exit(0)
if not form_input['variable'] or not form_input['station_ids']:
print 'variable and station_id options required!'
sys.exit(0)
s_date, e_date = WRCCUtils.start_end_date_to_eight(form_input)
station_ids = form_input['station_ids'] #list of stn ids (converted to list in form)
#sort coop ids in ascending order, strip left zeros first, sort and reattach zeros
station_ids = WRCCUtils.strip_n_sort(station_ids)
datadict = defaultdict(list)
station_names=[' ' for i in range(len(station_ids))]
if form_input['variable']!= 'multi':
variables = [form_input['variable']]
#evap, wdmv, wesf not fully implemented into Acis_WS yet
if form_input['variable'] in ['evap', 'wdmv', 'wesf']:
print 'Evaporation, wind and water equivalent not implemented yet. Please chose another variable!'
sys.exit(0)
else:
variables = ['pcpn', 'snow', 'snwd', 'maxt', 'mint', 'obst']
#request data on a station by station basis
for i, stn_id in enumerate(station_ids):
if form_input['variable']!= 'multi':
params = dict(sid=stn_id, sdate=s_date, edate=e_date, elems=[dict(name='%s' % form_input['variable'])])
else:
params = dict(sid=stn_id, sdate=s_date, edate=e_date, elems=[dict(name='pcpn'), \
dict(name='snow'), dict(name='snwd'), dict(name='maxt'), dict(name='mint'), dict(name='obst')])
request = StnData(params)
if not request:
request = {'error':'Bad request, check parameters.'}
try:
request['meta']
station_names[i] = request['meta']['name']
except:
station_names[i] = ' '
try:
request['data']
datadict[i] = request['data']
except:
datadict[i]=[]
return datadict, variables, station_ids, station_names
def get_sodlist_data(form_input, program):
'''
Data acquisition for sodlist,sodmonline(my), sodcnv
Keyword arguments:
form_input -- parameter file for data request obtained from user of WRCC SOD pages
program -- specifies program that is making the request.
'''
s_date, e_date = WRCCUtils.start_end_date_to_eight(form_input)
station_id = form_input['station_id']
if program in ['sodlist', 'sodcnv']:
if 'include_tobs_evap' in form_input.keys():
params = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, \
elems=[dict(name='pcpn', add='f,t'), dict(name='snow', add='f,t'), dict(name='snwd', add='f,t'),
dict(name='maxt', add='f,t'), dict(name='mint', add='f,t'), dict(name='obst', add='f,t')])
params_e = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, elems=[dict(vX=7,add='f,t', prec=2)])
else:
if not program == 'sodcnv':
params = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, \
elems=[dict(name='pcpn', add='f,t'), dict(name='snow', add='f,t'), dict(name='snwd', add='f,t'),
dict(name='maxt', add='f,t'), dict(name='mint', add='f,t')])
else:
params = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, \
elems=[dict(name='pcpn', add='f,t'), dict(name='snow', add='f,t'), dict(name='snwd', add='f,t'),
dict(name='maxt', add='f,t'), dict(name='mint', add='f,t'), dict(name='obst', add='f,t')])
elif program in ['sodmonline', 'sodmonlinemy']:
#sodmonline(my) only available for full years
s_date = '%s%s' % (s_date[0:4], '0101')
e_date = '%s%s' % (e_date[0:4], '1231')
if form_input['variable'] == 'evap':
vXvN = 7
elif form_input['variable'] == 'wdmv':
vXvN = 12
elif form_input['variable'] in ['wesf']:
vXvN = 13
if form_input['variable'] in ['evap','wdmv', 'wesf' ]: #need to work with var major (vX) and var minor (vN)
params = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, elems=[vXvN])
elif form_input['variable'] in ['dtr', 'mmt']:
params = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, elems=[dict(name='maxt'), dict(name='mint')])
else:
params = dict(sid='%s' % station_id, sdate=s_date, edate=e_date, elems=[dict(name='%s' % form_input['variable'])])
else:
print 'Program %s not supported in get_sodlist_data. Program should be one out of [sodlist, sodcnv, sodmonline, sodmonlinemy]!' % program
sys.exit(0)
#Request evap, wind and water equivalent data
#NOTE: these data need to be obtained via var major:
if program == 'sodlist' and 'include_tobs_evap' in form_input.keys():
request_evap = StnData(params_e)
if not request_evap:
request_evap = {'error':'Bad request, check parameters.'}
try:
request_evap['data']
evap_data = request_evap['data']
except:
evap_data = None
else:
evap_data = None
#retrieve data via Acis webservices
request = StnData(params)
if not request:
request = {'error':'Bad request, check parameters.'}
#Test for successful data retrieval and get metadata information
try:
request['meta']
stn_name = request['meta']['name']
except:
if request['error']:
stn_name = ' '
try:
request['data']
req_data = request['data']
except:
req_data = None
#get windowed data is program is sodlist
if program == 'sodlist':
if 'start_window' not in form_input.keys():
s_window = '0101'
else:
s_window = form_input['start_window']
if 'end_window' not in form_input.keys():
e_window = '1231'
else:
e_window = form_input['end_window']
if s_window!= '0101' or e_window != '1231':
if req_data:
data = WRCCUtils.get_windowed_data(req_data, s_date, e_date, s_window, e_window)
else:
data = None
else:
if req_data:
data = req_data
else:
data = None
else:
if req_data:
data = req_data
else:
data = None
#Join evap_data if present
if evap_data:
for i, val in enumerate(evap_data):
data[i].append(val[1])
return data, stn_name
def get_grid_data(form_input, program):
'''
Retrieves Grid Data from ACIS.
Keyword arguments:
form_input -- parameter file for data request obtained from user of CSC pages
program -- specifies program that is making the request.
Returns python dictionary with keys:
meta
data or smry (if data_summary)
lats -- lists of latitudes for grid points
lons -- lists of longitudes for grid points
'''
#datalist[date_idx] = [[date1,lat1, lon1, elev1, el1_val1, el2_val1, ...],
#[date2, lat2, ...], ...]
s_date, e_date = WRCCUtils.start_end_date_to_eight(form_input)
#grid data calls do not except list of variables, need to be string of comma separated values
el_list = WRCCUtils.get_variable_list(form_input, program)
if 'data_summary' in form_input.keys() and form_input['data_summary'] == 'temporal_summary':
variables = [{'name':str(el),'smry':str(form_input['temporal_summary']),'smry_only':1} for el in el_list]
else:
variables = ','.join(el_list)
params = {
'sdate': s_date,
'edate': e_date,
'grid': form_input['grid'],
'elems': variables,
'meta': 'll,elev'
}
#Set area parameter
try:
key = WRCCData.GRID_AREA_FORM_TO_PARAM[form_input['select_grid_by']]
val = form_input[form_input['select_grid_by']]
except:
key = WRCCData.GRID_AREA_FORM_TO_PARAM[form_input['area_type']]
val = form_input[form_input['area_type']]
params[key] = val
#Override area parameter if needed
#Find bbox if custom shape
if 'shape' in form_input.keys():
#Need to find enclosing bbox
shape_type,bbox = WRCCUtils.get_bbox(form_input['shape'])
if shape_type == 'location':params['loc'] = form_input['shape']
else:params['bbox'] = bbox
#Find enclosing bbox via General ACIS call if climdiv, cwa, county, basin
#Note: gridACIS currently does not support direct calls for these options
#FIX me : when gridACIS supports calls for climdiv, cwa, county, basin
if form_input['select_grid_by'] in ['county_warning_area', 'climate_division', 'basin', 'county']:
try:
a = WRCCData.STN_AREA_FORM_TO_PARAM[form_input['select_grid_by']]
v = form_input[form_input['select_grid_by']]
except:
a = WRCCData.STN_AREA_FORM_TO_PARAM[form_input['area_type']]
v = form_input[form_input['area_type']]
bbox = get_acis_bbox_of_area(a,v)
params['bbox'] = bbox
request = {}
try:
request = GridData(params)
except Exception, e:
error = 'GridData request failed with ERROR: %s.' %(str(e))
request['error'] = error
def get_sod_data(form_input, program):
'''
Data acquisition for Soddyrec, Soddynorm, Soddd, Sodpad, Sodsumm
Keyword arguments:
form_input -- parameter file for data request obtained from user of WRCC SOD pages
program -- specifies program that is making the request.
'''
s_date, e_date = WRCCUtils.start_end_date_to_eight(form_input)
'''
if program in ['Sodpiii']:
#get an extra year of data previos and after s_date, e_date
s_date = str(int(s_date[0:4]) - 1) + s_date[4:]
e_date = str(int(e_date[0:4]) + 1) + e_date[4:]
'''
dates = WRCCUtils.get_dates(s_date, e_date, program)
variables = WRCCUtils.get_variable_list(form_input, program)
els = [dict(name='%s' % el) for el in variables]
if 'station_id' in form_input.keys():
station_ids =[form_input['station_id']]
elif 'station_ids' in form_input.keys():
station_ids = form_input['station_ids']
elif 'county' in form_input.keys():
station_ids = get_station_list('county', form_input['county'])
elif 'climate_division' in form_input.keys():
station_ids = get_station_list('climate_division', form_input['climate_division'])
elif 'county_warning_area' in form_input.keys():
station_ids = get_station_list('county_warning_area', form_input['county_warning_area'])
elif 'basin' in form_input.keys():
station_ids = get_station_list('basin', form_input['basin'])
elif 'state' in form_input.keys():
station_ids = get_station_list('state', form_input['state'])
elif 'bounding_box' in form_input.keys():
station_ids = get_station_list('bounding_box', form_input['bounding_box'])
else:
station_ids =[]
#sort station id in ascending order
#station_ids = WRCCUtils.strip_n_sort(station_ids)
station_names=['No Name found' for i in range(len(station_ids))]
#Since Acis may not return results for some stations, we need to initialize output dictionary
datadict = defaultdict(list)
for i, stn in enumerate(station_ids):
if program == 'Soddyrec':
#yr_list = [[['#', '#', '#', '#', '#', '#','#', '#'] for k in range(366)] for el in variables]
#yr_list = [[['#', '#', '#', '#', '#', '#','#', '#'] for k in range(366)] for i in range(3*len(variables))]
#datadict[i] = yr_list
datadict[i] = []
elif program in ['Sodrun', 'Sodrunr']:
datadict[i] = []
else:
datadict[i] = [[] for el in variables]
if program == 'Soddyrec':
smry_opts = [{'reduce':'mean', 'add':'date,mcnt'}, {'reduce':'max', 'add':'date,mcnt'}, {'reduce':'min', 'add':'date,mcnt'}]
elts = []
for el in variables:
for sry in smry_opts:
#elts.append(dict(name=str(el),smry=sry, groupby='year'))
elts.append(dict(vX=WRCCData.ACIS_ELEMENTS_DICT[el]['vX'],smry=sry, groupby='year'))
params = dict(sids=station_ids, sdate=s_date, edate=e_date, elems=elts)
elif program in ['Soddynorm', 'Soddd', 'Sodpad', 'Sodsumm', 'Sodpct', 'Sodthr', 'Sodxtrmts', 'Sodpiii']:
params = dict(sids=station_ids, sdate=s_date, edate=e_date, \
#elems=[dict(name=el,interval='dly',duration='dly',groupby='year')for el in variables])
elems=[dict(vX=WRCCData.ACIS_ELEMENTS_DICT[el]['vX'],interval='dly',duration='dly',groupby='year') for el in variables])
elif program in ['Sodlist', 'Sodcnv']:
params = dict(sids=station_ids, sdate=s_date, edate=e_date, \
#elems=[dict(name=el,add='t')for el in variables])
elems=[dict(vX=WRCCData.ACIS_ELEMENTS_DICT[el]['vX'],add='t') for el in variables])
else:
params = dict(sids=station_ids, sdate=s_date, edate=e_date, \
#elems=[dict(name=el)for el in variables])
elems=[dict(vX=WRCCData.ACIS_ELEMENTS_DICT[el]['vX']) for el in variables])
request = MultiStnData(params)
if not request:
request = {'error':'bad request, check params: %s' % str(params)}
try:
request['data']#list of data for the stations
except:
if request['error']:
return datadict, dates, variables, station_ids, station_names
#sys.exit(1)
else:
#Unknown error ocurred when getting data
#sys.exit(1)
return datadict, dates, variables, station_ids, station_names
for stn, stn_data in enumerate(request['data']):
if not 'meta' in stn_data.keys():continue
sids = stn_data['meta']['sids']
index = None
for sid in sids:
sid_split = sid.split(' ')
station_id = str(sid_split[0])
try:
index = station_ids.index(str(station_id))
break
except:
continue
if index is None:continue
station_names[index] = str(stn_data['meta']['name'])
if program == 'Soddyrec':
try:
stn_data['smry']
datadict[index] = stn_data['smry']
except:
datadict[index] = []
#sort data by variable
elif program in ['Soddynorm', 'Soddd', 'Sodpct']:
try:
stn_data['data']
for yr, el_data in enumerate(stn_data['data']):
for el_idx, dat in enumerate(el_data):
datadict[index][el_idx].append(dat)
except:
pass
else:
try:
stn_data['data']
datadict[index] = stn_data['data']
except:
datadict[index] = []
'''
try:
stn_data['meta']
#find station_id, Note: MultiStnData call may not return the stations in order
sids = stn_data['meta']['sids']
for sid in sids:
sid_split = sid.split(' ')
station_id = str(sid_split[0])
if sid_split[1] == '2':
try:
index = station_ids.index(str(station_id))
break
except:
continue
index = station_ids.index(str(station_id))
try:
index = station_ids.index(str(station_id))
station_names[index] = str(stn_data['meta']['name'])
if program == 'Soddyrec':
try:
stn_data['smry']
datadict[index] = stn_data['smry']
except:
datadict[index] = []
#sort data by variable
elif program in ['Soddynorm', 'Soddd', 'Sodpct']:
try:
stn_data['data']
for yr, el_data in enumerate(stn_data['data']):
for el_idx, dat in enumerate(el_data):
datadict[index][el_idx].append(dat)
except:
pass
else:
try:
stn_data['data']
datadict[index] = stn_data['data']
except:
datadict[index] = []
except ValueError:
continue
except:
pass
'''
'''
if program == 'Soddyrec':
#need to get averages separately; add: date, mcnt fails if we ask for mean, max together
elts_x = [dict(name='%s' % el, interval='dly', duration='dly', smry={'reduce':'mean'}, \
groupby="year") for el in variables]
params_x = dict(sids=station_ids, sdate=s_date, edate=e_date, elems=elts_x)
request_x = MultiStnData(params_x)
if not request_x:
request_x = {'error':'Bad request, check parameters.'}
try:
request_x['data']#list of data for the stations
#order results by stn id
for stn, stn_data in enumerate(request_x['data']):
sids = stn_data['meta']['sids']
station_id = ' '
for sid in sids:
sid_split = sid.split(' ')
if sid_split[1] == '2':
#station_id = str(stn_data['meta']['sids'][1].split()[0])
station_id = str(sid_split[0])
break
if station_id == ' ':
continue
try:
indx_x = station_ids.index(station_id)
try:
for el_id, ave_list in enumerate(stn_data['smry']):
for date_id,ave in enumerate(ave_list):
datadict[indx_x][el_id][date_id].insert(0,ave)
#figure out the number of years that are in the record
val_list = [stn_data['data'][yr][el_id][date_id] for yr in range(len(stn_data['data']))]
yrs_in_record = len(filter(lambda a: a != 'M', val_list))
datadict[indx_x][el_id][date_id].append(yrs_in_record)
except:
pass
except ValueError:
continue
except:
if 'error' in request_x.keys():
print '%s' % str(request_x['error'])
sys.exit(1)
else:
'Unknown error ocurred when getting data'
sys.exit(1)
'''
return datadict, dates, variables, station_ids, station_names
def get_station_data(form_input, program):
'''
Retrieves Station Data from ACIS.
Keyword arguments:
form_input -- parameter file for data request obtained from user of CSC pages
program -- specifies program that is making the request.
Returns python dictionary with keys:
meta
data
'''
request = {'data':[]}
s_date, e_date = WRCCUtils.start_end_date_to_eight(form_input)
#Sanity check for valid date input:
if (s_date.lower() == 'por' or e_date.lower() == 'por') and ('station_id' not in form_input.keys()):
error = 'Parameter error. Start/End date ="por" not supported for multi station call.'
resultsdict['error'] = error
return resultsdict
variables = WRCCUtils.get_variable_list(form_input, program)
elems_list = []
elems_list_short = []
resultsdict = {}
for el in variables:
el_strip, base_temp = WRCCUtils.get_el_and_base_temp(el)
elems_list_short.append(el_strip)
if el_strip in ['gdd', 'hdd', 'cdd'] and base_temp is not None:
d = dict(vX=WRCCData.ACIS_ELEMENTS_DICT[el_strip]['vX'], base=int(base_temp), add='f,t')
elems_list.append(d)
else:
d = dict(vX=WRCCData.ACIS_ELEMENTS_DICT[el]['vX'],add='f,t')
elems_list.append(d)
params = {
'sdate':s_date,
'edate':e_date,
'meta':'name,state,sids,ll,elev,uid,valid_daterange',
'elems':elems_list
}
shape_type = None
#Deal with POR input dates
if 'station_id' in form_input.keys():
sid = form_input['station_id']
s = s_date.lower()
e = e_date.lower()
l = elems_list_short
vd, no_vd_els = WRCCUtils.find_valid_daterange(sid, start_date=s, end_date=e, el_list=l, max_or_min='max')
params['sdate'] = v[0]; params['edate'] = v[1]
if params['sdate'] == '9999-99-99' or params['edate'] == '9999-99-99':
error = 'No start/end date could be found for this station in the metadata database.'
resultsdict['error'] = error
return resultsdict
#Set area parameter
try:
key = WRCCData.STN_AREA_FORM_TO_PARAM[form_input['select_stations_by']]
val = form_input[form_input['select_stations_by']]
except:
key = WRCCData.STN_AREA_FORM_TO_PARAM[form_input['area_type']]
val = form_input[form_input['area_type']]
params[key] = val
#Find bbox if custom shape and update params['bbox']
if 'shape' in form_input.keys():
shape_type,bbox = WRCCUtils.get_bbox(form_input['shape'])
params['bbox'] = bbox
#Data request
try:
req = MultiStnData(params)
except Exception, e:
request['error'] = 'StnData request failed. Error: %s. Pameters: %s.' %(str(e), params)
return request