def set_min_max_dates(initial):
sd = '9999-99-99';ed = '9999-99-99'
sd_fut = sd; ed_fut = ed
if 'station_id' in initial.keys():
stn_json = settings.MEDIA_DIR + '/json/US_station_id.json'
stn_id, stn_name = WRCCUtils.find_id_and_name(initial['station_id'],stn_json)
els = []
if 'element' in initial.keys():
els = [initial['element']]
if initial['element'] in ['dtr','pet']:els = ['maxt','mint']
if 'elements' in initial.keys():
els = initial['elements']
if 'dtr' in els and 'maxt' not in els:
els.append('maxt')
if 'dtr' in els and 'mint' not in els:
els.append('mint')
if 'dtr' in els and 'maxt' not in els:
els.append('maxt')
if 'dtr' in els and 'mint' not in els:
els.append('mint')
vd = WRCCUtils.find_valid_daterange(stn_id,el_list=els,max_or_min='min')
if vd and len(vd) >=1:sd = vd[0]
if vd and len(vd) >1:ed = vd[1]
sd_fut = sd;ed_fut = ed
if 'location' in initial.keys() or initial['app_name'] == 'monthly_spatial_summary':
sd = WRCCData.GRID_CHOICES[str(initial['grid'])][3][0][0]
ed = WRCCData.GRID_CHOICES[str(initial['grid'])][3][0][1]
sd_fut = sd;ed_fut = ed
if len(WRCCData.GRID_CHOICES[initial['grid']][3]) == 2:
sd_fut = WRCCData.GRID_CHOICES[initial['grid']][3][1][0]
ed_fut = WRCCData.GRID_CHOICES[initial['grid']][3][1][1]
return sd, ed, sd_fut, ed_fut
def station_invalid(el_list, vX_list, time_range, stn, contraints):
#Check if constraints are met for element list and date range
if constraints in ['any_any', 'any_all']:
flag_invalid_station = True
elif constraints in ['all_all', 'all_any']:
flag_invalid_station = False
for el_idx, el_vX in enumerate(el_list):
#Find correct index in vX_list
try:
idx = vX_list.index(el_vX)
except:
if constraints in ['all_all', 'all_any']:
flag_invalid_station = True
break
elif constraints in ['any_any', 'any_all']:
continue
#Sanity Check
if not stn['valid_daterange'][idx] and (constraints == 'all_all' or constraints == 'all_any' or constraints is None):
#data for this element does not exist at station
flag_invalid_station = True
break
elif not stn['valid_daterange'][idx] and (constraints == 'any_any' or constraints == 'any_all'):
continue
#Find period of record for this element and station
por_start = WRCCUtils.date_to_datetime(stn['valid_daterange'][idx][0])
por_end = WRCCUtils.date_to_datetime(stn['valid_daterange'][idx][1])
if time_range[0].lower() != 'por':
user_start = WRCCUtils.date_to_datetime(time_range[0])
else:
user_start = por_start
if time_range[1].lower() != 'por':
user_end = WRCCUtils.date_to_datetime(time_range[1])
else:
user_end = por_end
#Check constraints logic for this element and station
if constraints == 'all_all' or constraints is None:
#all elements have data records for all dates within start and end date given by user
if user_start < por_start or user_end > por_end:
flag_invalid_station = True
break
elif constraints == 'any_any':
#At least one element has one data record within user given time_range
if (user_end >= por_start and user_start <= por_end) or (user_start <= por_end and user_end >=por_start):
flag_invalid_station = False
break
elif constraints == 'all_any':
#All elements have at least one data record within user given time_range
if (user_end >= por_start and user_start <= por_end) or (user_start <= por_end and user_end >=por_start):
continue
else:
flag_invalid_station = True
break
elif constraints == 'any_all':
#At least one elements has data records for all dates within given date_range
if user_start >= por_start and user_end <= por_end:
flag_invalid_station = False
break
return flag_invalid_station
def set_min_max_dates(initial):
sd = '9999-99-99';ed = '9999-99-99'
sd_fut = sd; ed_fut = ed
data_type = 'station'
if 'location' in initial.keys():
data_type = 'grid'
if initial['app_name'] in ['monthly_spatial_summary', 'temporal_summary']:
data_type = 'grid'
if 'data_type' in initial.keys() and initial['data_type'] == 'grid':
data_type = 'grid'
if 'station_id' in initial.keys():
stn_json = settings.MEDIA_DIR + '/json/US_station_id.json'
stn_id, stn_name = WRCCUtils.find_id_and_name(initial['station_id'],stn_json)
els = []
if 'variable' in initial.keys():
els = [initial['variable']]
if 'variables' in initial.keys():
els = initial['variables']
vd, no_vd_els = WRCCUtils.find_valid_daterange(stn_id,el_list=els,max_or_min='min')
sd = vd[0];ed = vd[1]
#sd_fut = sd;ed_fut = ed
elif data_type == 'grid':
sd = WRCCData.GRID_CHOICES[str(initial['grid'])][3][0][0]
#ed = WRCCUtils.advance_date(sd,10*365,'forward')
ed = WRCCData.GRID_CHOICES[str(initial['grid'])][3][0][1]
#sd_fut = sd;ed_fut = ed
if len(WRCCData.GRID_CHOICES[initial['grid']][3]) == 2:
sd_fut = WRCCData.GRID_CHOICES[initial['grid']][3][1][0]
#ed_fut = WRCCUtils.advance_date(sd,10*365,'forward')
ed_fut = WRCCData.GRID_CHOICES[initial['grid']][3][1][1]
else:
if 'data_type' in initial.keys() and initial['data_type'] == 'station':
sd = '1850-01-01'
ed = today
return sd, ed, sd_fut, ed_fut
def get_sodsum_data(form_input):
'''
Data acquisition for sodsum
Keyword arguments:
form_input -- parameter file for data request obtained from user of WRCC SOD pages
'''
if 'variable' not in form_input.keys() or 'station_ids' not in form_input.keys():
print 'variable and station_id options required!'
sys.exit(0)
if not form_input['variable'] or not form_input['station_ids']:
print 'variable and station_id options required!'
sys.exit(0)
s_date, e_date = WRCCUtils.start_end_date_to_eight(form_input)
station_ids = form_input['station_ids'] #list of stn ids (converted to list in form)
#sort coop ids in ascending order, strip left zeros first, sort and reattach zeros
station_ids = WRCCUtils.strip_n_sort(station_ids)
datadict = defaultdict(list)
station_names=[' ' for i in range(len(station_ids))]
if form_input['variable']!= 'multi':
variables = [form_input['variable']]
#evap, wdmv, wesf not fully implemented into Acis_WS yet
if form_input['variable'] in ['evap', 'wdmv', 'wesf']:
print 'Evaporation, wind and water equivalent not implemented yet. Please chose another variable!'
sys.exit(0)
else:
variables = ['pcpn', 'snow', 'snwd', 'maxt', 'mint', 'obst']
#request data on a station by station basis
for i, stn_id in enumerate(station_ids):
if form_input['variable']!= 'multi':
params = dict(sid=stn_id, sdate=s_date, edate=e_date, elems=[dict(name='%s' % form_input['variable'])])
else:
params = dict(sid=stn_id, sdate=s_date, edate=e_date, elems=[dict(name='pcpn'), \
dict(name='snow'), dict(name='snwd'), dict(name='maxt'), dict(name='mint'), dict(name='obst')])
request = StnData(params)
if not request:
request = {'error':'Bad request, check parameters.'}
try:
request['meta']
station_names[i] = request['meta']['name']
except:
station_names[i] = ' '
try:
request['data']
datadict[i] = request['data']
except:
datadict[i]=[]
return datadict, variables, station_ids, station_names
def stn_in_poly(by_type, shape_type, shape,stn_meta):
#convert to floats
s = [float(s) for s in shape]
if shape_type == 'circle':
poly = s
stn_in = WRCCUtils.point_in_circle(stn_meta['ll'][0], stn_meta['ll'][1], poly)
else:
if shape_type in ['bbox','location']:
poly = [(s[0],s[1]), (s[0],s[3]),(s[2],s[3]),(s[2],s[1])]
else:
poly = [(s[2*idx],s[2*idx+1]) for idx in range(len(s)/2)]
stn_in = WRCCUtils.point_in_poly(stn_meta['ll'][0], stn_meta['ll'][1], poly)
if not stn_in:return False
else:return True
def check_end_window(form):
#Backbutton sanity check
if 'start_window' not in form.keys() or 'end_window' not in form.keys():
return 'Start and/or End Window field not valid. You may have pressed the backbutton. Please reset the windowed data button.'
err = None
s_w = form['start_window'].replace('-','').replace('/','').replace(':','')
e_w = form['end_window'].replace('-','').replace('/','').replace(':','')
if len(e_w) != 4:
return 'Start Window must be of form mmdd/mm-dd or mm:dd. You entered %s' % form['start_window']
mm = e_w[0:2]
dd= e_w[2:4]
#Check month
if int(mm) < 1 or int(mm) > 12:
return 'Not a valid month.'
#Check day
if int(dd) < 1 or int(dd) > 31:
return 'Not a valid day.'
ml = WRCCData.MONTH_LENGTHS[int(mm) - 1]
if int(dd) > ml:
if mm == '02' and dd == '29':
if 'end_date' in form.keys() and WRCCUtils.is_leap_year(form['end_date'][0:4]):
return err
else:
return 'Month %s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[mm],ml,dd)
else:
return 'Month %s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[mm],ml,dd)
return err
def create_kml_file(area_type, overlay_state):
kml_file_name = overlay_state + '_' + area_type + '.kml'
kml_file_path = settings.TMP_URL + kml_file_name
status = WRCCUtils.generate_kml_file(area_type, overlay_state, kml_file_name, settings.TEMP_DIR)
if status != 'Success':
return 'ERROR: ' + status
return kml_file_path
def set_wrapper_params(stn_id):
'''
yesterday = WRCCUtils.set_back_date(1)
w_params = [stn_id, 'all','18000101',yesterday]
'''
vd = WRCCUtils.find_valid_daterange(stn_id)
if len(vd) == 2 and vd[0] and vd[1]:
return [stn_id, 'all',vd[0],vd[1]]
return []
def test_find_id_and_name(self):
for area_type, area_val in self.test_areas.iteritems():
if area_type in ['station_id','basin','county','county_warning_area','climate_division']:
json_file_path = '/www/apps/csc/dj-projects/my_acis/media/json/US_' + area_type + '.json'
ID, name = WRCCUtils.find_id_and_name(area_val, json_file_path)
try:
self.assertNotEqual('ID','')
except AssertionError as err:
logger.error('AssertionError' + str(err))
def test_get_el_and_base_temp(self):
for unit in self.units:
for el in self.variables:
el, base_temp = WRCCUtils.get_el_and_base_temp(el,units=unit)
self.assertIsInstance(el, basestring)
self.assertIn(el,WRCCData.ACIS_ELEMENTS_DICT.keys())
try:
int(base_temp)
except:
self.assertEqual(base_temp,None)
def run_multi_lister(self,params):
results = {};err = None
initial, err = self.setInitial(params)
if err is not None:return results, err
form, err = self.setForm(initial)
if err is not None:return results, err
form_cleaned, err = self.setFormCleaned(initial)
if err is not None:return results, err
try:
results = WRCCUtils.request_and_format_data(form_cleaned)
except Exception, e:
err = 'FAIL request_and_format_data. ERROR: ' + str(e) + ' PARAMS: ' + str(params)
return results, err
def test_find_ids_and_names(self):
area_type = 'station_ids'
area_val = self.test_areas[area_type]
json_file_path = '/www/apps/csc/dj-projects/my_acis/media/json/US_station_id.json'
IDs, names = WRCCUtils.find_ids_and_names(area_val, json_file_path)
try:
self.assertIsInstance(IDs, str)
except AssertionError as err:
logger.error('AssertionError' + str(err))
try:
self.assertNotEqual(IDs.split(','),[])
except AssertionError as err:
logger.error('AssertionError' + str(err))
def run_seasonal_summary(self,params):
results = {};err = None
initial, err = self.setInitial(params)
if err is not None:return results, err
form, err = self.setForm(initial)
if err is not None:return results, err
form_cleaned, err = self.setFormCleaned(initial)
if err is not None:return results, err
try:
year_data, hc_data = WRCCUtils.get_single_seasonal_summary_data(form_cleaned)
except Exception, e:
err = 'FAIL get_single_seasonal_summary_data. ERROR: ' + \
str(e) + ' PARAMS: ' + str(params)
return results, err
def run_station_finder(self, form_cleaned):
vX_list = []
for el_idx, variable in enumerate(form_cleaned['variables']):
el,base_temp = WRCCUtils.get_el_and_base_temp(variable)
vX_list.append(str(WRCCData.ACIS_ELEMENTS_DICT[el]['vX']))
by_type = WRCCData.ACIS_TO_SEARCH_AREA[form_cleaned['area_type']]
val = form_cleaned[WRCCData.ACIS_TO_SEARCH_AREA[form_cleaned['area_type']]]
dr = [form_cleaned['start_date'],form_cleaned['end_date']]
ec = form_cleaned['variables_constraints']
dc = form_cleaned['dates_constraints']
edc = ec + '_' + dc
station_json, f_name = AcisWS.station_meta_to_json(by_type, val, el_list=vX_list,time_range=dr, constraints=edc)
return station_json, f_name
def run_single_year(self,params):
results = {};err = None
initial, err = self.setInitial(params)
if err is not None:return results, err
form, err = self.setForm(initial)
if err is not None:return results, err
form_cleaned, err = self.setFormCleaned(initial)
if err is not None:return results, err
try:
year_txt_data, year_graph_data, climoData, percentileData =\
WRCCUtils.get_single_single_year_data(form_cleaned)
except Exception, e:
err = 'FAIL get_single_intraanual_data. ERROR: ' + \
str(e) + ' PARAMS: ' + str(params)
return results, err
def run_station_finder(self, form_cleaned):
vX_list = []
for el_idx, element in enumerate(form_cleaned["elements"]):
el, base_temp = WRCCUtils.get_el_and_base_temp(element)
vX_list.append(str(WRCCData.ACIS_ELEMENTS_DICT[el]["vX"]))
by_type = WRCCData.ACIS_TO_SEARCH_AREA[form_cleaned["area_type"]]
val = form_cleaned[WRCCData.ACIS_TO_SEARCH_AREA[form_cleaned["area_type"]]]
dr = [form_cleaned["start_date"], form_cleaned["end_date"]]
ec = form_cleaned["elements_constraints"]
dc = form_cleaned["dates_constraints"]
edc = ec + "_" + dc
station_json, f_name = AcisWS.station_meta_to_json(
by_type, val, el_list=vX_list, time_range=dr, constraints=edc
)
return station_json, f_name
def compose_email(params, ftp_server, ftp_dir, out_files):
#NOTIFY_USER
mail_server = settings.DRI_MAIL_SERVER
fromaddr = settings.CSC_FROM_ADDRESS
user_name, user_email = get_user_info(params)
subj = 'Data request %s' % params['output_file_name']
now = datetime.datetime.now()
date = now.strftime( '%Y-%m-%d %H:%M' )
pick_up_latest = (now + datetime.timedelta(days=25)).strftime( '%Y-%m-%d' )
display_keys = [params['area_type'],'variables','units', 'start_date', 'end_date']
if 'data_type' in params.keys():
display_keys.insert(0,'data_type')
params_display_list = WRCCUtils.form_to_display_list(display_keys, params)
dp = '';files=''
for item in params_display_list:
key = item[0]
val = item[1]
dp+=key + ': ' + val +'\n' + ' '
zip_file = out_files[0].split('/')[-1]
for f in out_files[1:]:
files+= f + '\n' + ' '
message_text ='''
Date: %s
Dear %s,
Your data request has been processed.
^ ^
(O,O)
( )
-"-"-
The data is available here:
%s
Please connect as Guest. You will not need a password.
The data is stored in the zip archive:
%s
The individual file names are:
%s
You can pick up your data until: %s.
Your parameters were:
%s
'''%(date, user_name,'ftp://' + ftp_server + ftp_dir, zip_file, files, str(pick_up_latest), dp)
return subj, message_text
def start_logger(base_dir):
#Set up Logging
#Start new log file for each day:
today = datetime.datetime.today()
day_stamp = WRCCUtils.datetime_to_date(today,'')
#day_stamp = '%s%s%s' %(str(today.year), str(today.month), str(today.day))
log_file_test = 'scenic_data_requests_' + day_stamp + '.log'
#Check if we need to create a new log file
#Look at most recent log file
log_files = sorted([ f for f in os.listdir(base_dir) if f.endswith('.log')])
if not log_files:log_file_name = ''
else:log_file_name = log_files[-1]
if log_file_test != log_file_name:log_file_name = log_file_test
#Start Logging
LOGGER = WRCCClasses.Logger(base_dir, log_file_name, 'scenic_data_requests')
logger = LOGGER.start_logger()
return logger, log_file_name
def test_defaults(self):
#Test most general case:
#convert all form aprameters to display
key_order_list = None
for app_name, params in self.test_params.iteritems():
display_params = WRCCUtils.form_to_display_list(key_order_list, params)
try:
self.assertIsInstance(display_params, list)
except AssertionError as err:
logger.error('AssertionError' + str(err))
try:
self.assertNotEqual(display_params,[])
except AssertionError as err:
logger.error('AssertionError' + str(err))
logger.info(display_params)
for dp in display_params:
try:
self.assertEqual(len(dp),2)
except AssertionError as err:
logger.error('AssertionError' + str(err))
def get_row_data(self):
if self.data:
return self.data
with open(self.json_in_file, 'r') as json_f:
json_data = WRCCUtils.u_convert(json.loads(json_f.read()))
'''
try:
with open(self.json_in_file, 'r') as json_f:
#need unicode converter since json.loads writes unicode
json_data = WRCCUtils.u_convert(json.loads(json_f.read()))
#json_data = json.loads(json_f.read())
#Find header info if in json_data
except:
json_data = {}
'''
#Set headers and column headers for the apps
if self.app_name == 'Sodxtrmts':
try:
self.header = json_data['header']
except:
pass
if self.app_name == 'Sodsumm':
self.header = []
labels = ['*','*Start Year', '*End Year']
for idx, key in enumerate(['title','record_start', 'record_end']):
self.header.append([labels[idx], json_data[key]])
if json_data['subtitle'] != '' and json_data['subtitle'] != ' ':
self.header.insert(1,['*', json_data['subtitle']])
if self.app_name == 'area_time_series':
self.header = json_data['display_params_list']
for el in json_data['search_params']['variable_list']:
self.column_headers['area_time_series'].append(el)
if self.app_name == 'spatial_summary':
self.header = json_data['params_display_list']
if self.app_data_dict[self.app_name] in json_data.keys():
data = json_data[self.app_data_dict[self.app_name]]
else:
data = []
return data
def test_find_area(self):
msg = 'Testing Area finder for Multi Lister'
logger.info(msg)
for area_type, area in self.area_dict.iteritems():
if area_type in ['station_id','station_ids','shape','location','state']:
continue
msg = 'Area: ' + area_type
logger.info(msg)
station_json = '/www/apps/csc/dj-projects/my_acis/media/json/US_'+area_type+'.json'
ID, name = WRCCUtils.find_id_and_name(area,station_json)
try:
self.assertIsInstance(name, str)
except AssertionError as err:
logger.error('AssertionError' + str(err))
try:
self.assertIsInstance(ID, str)
except AssertionError as err:
logger.error('AssertionError' + str(err))
try:
self.assertNotEqual(ID,'')
except AssertionError as err:
logger.error('AssertionError' + str(err))
def check_end_date(form):
err = None
s_date = form['start_date']
e_date = form['end_date']
#Check valid daterange error
if e_date.lower() == 'por':
if 'station_id' in form.keys():
return err
else:
return '%s is not a valid Start Date for a multi-station or grid request!' %str(form['end_date'])
s_date = WRCCUtils.date_to_eight(s_date)
e_date = WRCCUtils.date_to_eight(e_date)
if len(e_date)!=8:
return '%s is not a valid date.' %str(form['end_date'])
try:
int(e_date)
except:
return '%s is not a valid date.' %str(form['end_date'])
#Check month
if int(e_date[4:6]) < 1 or int(e_date[4:6]) > 12:
return '%s is not a valid date.' %str(form['end_date'])
#Check day
if int(e_date[6:8]) < 1 or int(e_date[4:6]) > 31:
return '%s is not a valid date.' %str(form['end_date'])
#Ceck for month length
ml = WRCCData.MONTH_LENGTHS[int(e_date[4:6]) - 1]
if int(e_date[6:8]) > ml:
if str(e_date[4:6]) == '02' or str(e_date[4:6]) == '2':
if int(e_date[6:8]) == 29 and WRCCUtils.is_leap_year(e_date[0:4]):
return None
#return '%s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[str(e_date[4:6])],'29',str(e_date[6:8]))
else:
return '%s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[str(e_date[4:6])],str(ml),str(e_date[6:8]))
else:
return '%s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[str(e_date[4:6])],str(ml),str(e_date[6:8]))
#Check that start date is ealier than end date
if s_date.lower() == 'por':
return err
try:
sd = datetime.datetime(int(s_date[0:4]), int(s_date[4:6].lstrip('0')), int(s_date[6:8].lstrip('0')))
except:
pass
try:
ed = datetime.datetime(int(e_date[0:4]), int(e_date[4:6].lstrip('0')), int(e_date[6:8].lstrip('0')))
except:
return '%s is not a valid date.' %form['end_date']
try:
if ed < sd:
return 'Start Date is later then End Year.'
except:pass
#Check grid data dates
if 'location' in form.keys() or ('data_type' in form.keys() and form['data_type'] == 'grid'):
flag = False
#Set grid date range
#Set grid date range
if int(form['grid']) in range(22,42):
grid_dr = [['19500101','20991231']]
else:
grid_dr = WRCCData.GRID_CHOICES[str(form['grid'])][3]
#For Prism we need to check if monthy/yearly resolution
#and pick proper daterange
if 'temporal_resolution' in form.keys() and form['temporal_resolution'] in ['mly','yly'] and str(form['grid']) == '21':
grid_dr = WRCCData.PRISM_MLY_YLY[str(form['grid'])][3]
for dr in grid_dr:
grid_s = WRCCUtils.date_to_datetime(dr[0])
grid_e = WRCCUtils.date_to_datetime(dr[1])
if grid_s <= sd and ed <= grid_e:
flag = False
break
else:
if str(dr[1]) == str(grid_dr[-1][1]):
flag = True
continue
if flag:
grid_s = WRCCUtils.datetime_to_date(grid_s,'-')
grid_e = WRCCUtils.datetime_to_date(grid_e,'-')
return 'Valid date range for this grid is: ' + grid_s + ' - ' + grid_e
'''
#Limit grid requests to max 10 years for multi point requests
if not 'location' in form.keys() and (ed - sd).days > 10 * 366:
err = 'Request for more than one grid point are limited to ten years or less! ' +\
'Please adjust your dates accordingly.'
return err
'''
#Check for unreasonable start and end dates
#for station data requests
data_type = WRCCUtils.get_data_type(form)
unreasonable = False
if data_type == 'station' and form['app_name'] == 'multi_lister':
#Limit multi station requests to 75 years
if not 'station_id' in form.keys() and (ed - sd).days > 75 * 366:
err = 'Request for more than one station are limited to 75 years or less! ' +\
'Please adjust your dates accordingly.'
#Check that station data end date is today or earlier
if 'station_id' in form.keys() or ('data_type' in form.keys() and form['data_type'] == 'station'):
if e_date.lower() != 'por':
today = WRCCUtils.format_date_string(WRCCUtils.set_back_date(0),'-')
if WRCCUtils.date_to_datetime(e_date) > WRCCUtils.date_to_datetime(today):
return 'Not a valid End Date. Date should be ' + today +' or ealier.'
return err
form_cleaned, err = self.setFormCleaned(initial)
if err is not None:
return results, err
try:
results = WRCCUtils.request_and_format_data(form_cleaned)
except Exception, e:
err = "FAIL request_and_format_data. ERROR: " + str(e) + " PARAMS: " + str(params)
return results, err
if "data_summary" in params.keys() and params["data_summary"] == "windowed_data":
d = copy.deepcopy(results["data"][0])
sd = params["start_date"]
ed = params["end_date"]
sw = params["start_window"]
ew = params["end_window"]
try:
results["data"] = WRCCUtils.get_windowed_data(d, sd, ed, sw, ew)
except Exception, e:
results = {}
err = "FAIL get_windowed_data. ERROR: " + str(e) + " PARAMS: " + str(params)
return results, err
return results, err
def test_single_lister_results(self, utClass, results, err):
if err is not None:
logger.error(err + "\n")
try:
utClass.assertIsNone(err)
except AssertionError as err:
logger.error("AssertionError " + str(err) + "\n")
try:
utClass.assertIsInstance(results, dict)
def set_wrapper_params(stn_id, table_name):
vd, no_vd_els = WRCCUtils.find_valid_daterange(stn_id)
if len(vd) == 2 and vd[0] != '9999-99-99' and vd[1] != '9999-99-99':
return [stn_id, table_name,vd[0][0:4],vd[1][0:4],5]
return []
def check_start_date(form):
err = None
s_date = form['start_date']
e_date = form['end_date']
if s_date.lower() == 'por':
if 'station_id' in form.keys():
return err
else:
return '%s is not a valid option for a multi-station or grid request.' %form['start_date']
s_date = WRCCUtils.date_to_eight(s_date)
e_date = WRCCUtils.date_to_eight(e_date)
#Valid daterange check
if len(s_date)!=8:
return '%s is not a valid date.' %str(form['start_date'])
try:
int(s_date)
except:
return '%s is not a valid date.' %str(form['start_date'])
#Check month
if int(s_date[4:6]) < 1 or int(s_date[4:6]) > 12:
return '%s is not a valid date.' %str(form['start_date'])
#Check day
if int(s_date[6:8]) < 1 or int(s_date[4:6]) > 31:
return '%s is not a valid date.' %str(form['start_date'])
#Check for month lengths
ml = WRCCData.MONTH_LENGTHS[int(s_date[4:6]) - 1]
if int(s_date[6:8]) > ml:
if str(s_date[4:6]) == '02' or str(s_date[4:6]) == '2':
if int(s_date[6:8]) == 29 and WRCCUtils.is_leap_year(s_date[0:4]):
return None
else:
return '%s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[str(s_date[4:6])],str(ml),str(s_date[6:8]))
else:
return '%s only has %s days. You entered: %s' %(WRCCData.NUMBER_TO_MONTH_NAME[str(s_date[4:6])],str(ml),str(s_date[6:8]))
#Check for leap year issue
if not WRCCUtils.is_leap_year(s_date[0:4]) and s_date[4:6] == '02' and s_date[6:8] == '29':
return '%s is not a leap year. Change start date to February 28.' %str(s_date[0:4])
#Check that start date is earlier than end date
if e_date.lower() == 'por':
return err
sd = '9999-99-99'
ed = '9999-99-99'
try:
sd = datetime.datetime(int(s_date[0:4]), int(s_date[4:6]), int(s_date[6:8]))
except:
return '%s is not a valid date.' %str(form['start_date'])
try:
ed = datetime.datetime(int(e_date[0:4]), int(e_date[4:6]), int(e_date[6:8]))
if ed < sd:
return 'Start Date is later then End Year.'
except:
#return 'End date %s is not a valid date.' %str(form['end_date'])
pass
#Check grid data dates
if 'location' in form.keys() or ('data_type' in form.keys() and form['data_type'] == 'grid'):
flag = False
#Set grid date range
#Set grid date range
if int(form['grid']) in range(22,42):
grid_dr = [['19500101','20991231']]
else:
grid_dr = WRCCData.GRID_CHOICES[str(form['grid'])][3]
#For Prism we need to check if monthy/yearly resolution
#and pick proper daterange
if 'temporal_resolution' in form.keys() and form['temporal_resolution'] in ['mly','yly'] and str(form['grid']) == '21':
grid_dr = WRCCData.PRISM_MLY_YLY[str(form['grid'])][3]
for dr in grid_dr:
grid_s = WRCCUtils.date_to_datetime(dr[0])
grid_e = WRCCUtils.date_to_datetime(dr[1])
if grid_s <= sd and ed <= grid_e:
flag = False
break
else:
if str(dr[1]) == str(grid_dr[-1][1]):
flag = True
continue
if flag:
grid_s = WRCCUtils.datetime_to_date(grid_s,'-')
grid_e = WRCCUtils.datetime_to_date(grid_e,'-')
return 'Valid date range for this grid is: ' +str(grid_s) + ' - ' + str(grid_e)
'''
#Limit grid requests to max 10 years for multi point requests
if not 'location' in form.keys() and (ed - sd).days > 10 * 366:
err = 'Request for more than one grid point are limited to ten years or less! ' +\
'Please adjust your dates accordingly.'
return err
'''
#Check for unreasonable start and end dates
#for station data requests
data_type = WRCCUtils.get_data_type(form)
unreasonable = False
if data_type == 'station':
#Limit multi station requests to 75 years
if not 'station_id' in form.keys() and (ed - sd).days > 75 * 366:
err = 'Request for more than one station are limited to 75 years or less! ' +\
'Please adjust your dates accordingly.'
return err
unreasonable = False
if s_date.lower() !='por' and int(s_date[0:4]) <= 1850:
unreasonable = True
if unreasonable:
meta_params = {
WRCCData.FORM_TO_META_PARAMS[form['area_type']]: form[form['area_type']],
'elems':','.join(form['variables']),
'meta':'valid_daterange'
}
if 'pet' in form['variables'] or 'dtr' in form['variables']:
meta_params['elems'].replace('pet','maxt,mint')
meta_params['elems'].replace('dtr','maxt,mint')
#meta_data = AcisWS.StnMeta(meta_params)
try:
meta_data = AcisWS.StnMeta(meta_params)
except:
meta_data = {'meta':[]}
start_dts = [];end_dts = []
if meta_data and 'meta' in meta_data.keys():
for stn_meta in meta_data['meta']:
for el_vd in stn_meta['valid_daterange']:
if el_vd and len(el_vd) == 2:
start_dts.append(WRCCUtils.date_to_datetime(el_vd[0]))
end_dts.append(WRCCUtils.date_to_datetime(el_vd[1]))
if start_dts and end_dts:
start = min(start_dts)
end = max(end_dts)
if start > WRCCUtils.date_to_datetime(s_date) and end < WRCCUtils.date_to_datetime(e_date):
s = WRCCUtils.datetime_to_date(start,'-')
e = WRCCUtils.datetime_to_date(end,'-')
err = 'Please change Start and End Date to earliest/latest record found: ' +\
s + ', ' + e
return err
if start_dts:
start = min(start_dts)
if start > WRCCUtils.date_to_datetime(s_date):
s = WRCCUtils.datetime_to_date(start,'-')
err = 'Please change Start Date to earliest record found: ' + s
return err
#Check station data start date for single station requesrs
if 'station_id' in form.keys():
if int(s_date[0:4]) < int(stn_earliest[0:4]):
return 'Not a valid Start Date. Year must be later than %s.' %(stn_earliest[0:4])
return err
#!/usr/bin/python
'''
Module WRCCFormCheck
Checks input form parameters
'''
import datetime
import re
import WRCCData, WRCCUtils, AcisWS
today = datetime.datetime.today()
stn_earliest = '18250101'
stn_earliest_dt = WRCCUtils.date_to_datetime(stn_earliest)
def check_start_year(form):
err = None
yr = form['start_year']
#Check for valid daterange error
if yr == '':
return 'No valid start year could be found for this station!'
if yr.lower() == 'por':
if 'location' in form.keys():
return 'POR is not a valid year for gridded data.'
return err
else:
return err
if len(yr)!=4:
return 'Year should be of form yyyy. You entered %s' %yr
try:
int(yr)
def set_initial(request,app_name):
'''
Set html form
Args:
request: django request object
app_name: application, one of
single_lister, multi_lister, station_finder
map_overlay,
sf_download
spatial_summary, temporal_summary
monthly_summary, climatology
data_comparison, liklihood,
data_download
Returns:
two dictionaries
initial: form input
'''
initial = {}
initial['app_name'] = app_name
Get = set_GET(request)
Getlist = set_GET_list(request)
#Set area type: station_id(s), location, basin,...
area_type = None
if app_name in ['single_lister','climatology','monthly_summary', 'seasonal_summary','intraannual']:
initial['area_type'] = Get('area_type','station_id')
elif app_name in ['data_comparison']:
initial['area_type'] = 'location'
else:
initial['area_type'] = Get('area_type','state')
#Set todays date parameters
initial['today_year'] = today_year
initial['today_month'] = today_month
initial['today_day'] = today_day
#Set area depending on area_type
if app_name == 'data_comparison':
location = Get('location',None)
station_id = Get('station_id',None)
if location is None and station_id is not None:
#Link from station finder,
#set location to station lon, lat if we are
stn_id, stn_name = WRCCUtils.find_id_and_name(station_id,settings.MEDIA_DIR + '/json/US_station_id.json')
meta = AcisWS.StnMeta({'sids':stn_id,'meta':'ll'})
ll = None
ll = str(meta['meta'][0]['ll'][0]) + ',' + str(meta['meta'][0]['ll'][1])
initial['location'] = ll
else:
initial[str(initial['area_type'])] = Get(str(initial['area_type']), WRCCData.AREA_DEFAULTS[str(initial['area_type'])])
else:
initial[str(initial['area_type'])] = Get(str(initial['area_type']), WRCCData.AREA_DEFAULTS[str(initial['area_type'])])
initial['area_type_label'] = WRCCData.DISPLAY_PARAMS[initial['area_type']]
initial['area_type_value'] = initial[str(initial['area_type'])]
#Set data type and map parameters
if initial['area_type'] in ['station_id','station_ids']:
initial['autofill_list'] = 'US_' + initial['area_type']
initial['data_type'] = 'station'
elif initial['area_type'] in ['location','locations']:
initial['data_type'] = 'grid'
elif initial['area_type'] in ['basin','county_warning_area','county','climate_division','state','shape']:
initial['autofill_list'] = 'US_' + initial['area_type']
initial['data_type'] = Get('data_type','station')
if app_name in ['temporal_summary','monthly_spatial_summary']:
initial['data_type'] = 'grid'
if app_name in ['station_finder','sf_download']:
initial['data_type'] = 'station'
#Grid
if app_name not in ['station_finder', 'sf_download']:
initial['grid'] = Get('grid','1')
#Set up map parameters
initial['overlay_state'] = Get('overlay_state','nv').lower()
initial['host'] = settings.HOST
#Create kml files for oerlay state
for at in ['basin', 'county', 'county_warning_area', 'climate_division']:
kml_file_path = create_kml_file(at, initial['overlay_state'])
if initial['area_type'] == at:
initial['kml_file_path'] = kml_file_path
#If station_finder download, we need to set the station_ids
#and override the original area type fields
if app_name == 'sf_download':
#delete old are type
del initial[str(initial['area_type'])]
#set new area params
initial['station_ids'] = str(Get('station_ids_string',''))
initial['station_ids_string'] = initial['station_ids']
initial['area_type'] = 'station_ids'
initial['area_type_label'] = 'Station IDs'
initial['area_type_value'] = initial['station_ids']
initial['station_json'] = Get('station_json','')
#If station finder set hidden var station_ids_string for results
if app_name == 'station_finder':
initial['station_ids_string'] = str(Get('station_ids_string',''))
#Set element(s)--> always as list if multiple
if app_name == 'map_overlay':
initial['elements'] = Get('elements','maxt,mint,pcpn').split(',')
initial['elements_str'] = ','.join(initial['elements'])
elif app_name in ['monthly_spatial_summary','monthly_summary','data_comparison', 'seasonal_summary','intraannual']:
initial['element'] = Get('element',None)
if initial['element'] is not None and len(initial['element'].split(',')) > 1:
initial['element'] = str(initial['element'].split(',')[0])
if initial['element'] is None:
#Link from station finder
initial['element'] = Get('elements','pcpn')
if len(initial['element'].split(',')) > 1:
initial['element'] = str(initial['element'].split(',')[0])
else:
els = Getlist('elements',None)
if not els:
els = Get('elements',None)
if not els:
els = ['maxt','mint','pcpn']
elif isinstance(els, basestring):
els = els.replace(' ','').split(',')
elif isinstance(els, list) and len(els) == 1 and len(els[0].split(',')) > 1:
els = els[0].replace(' ','').split(',')
elif isinstance(els, basestring):
els = els.replace(' ','').split(',')
initial['elements'] = [str(el) for el in els]
initial['elements_str'] = ','.join(initial['elements'])
#Set units
initial['units'] = Get('units','english')
#Set degree days
if app_name not in ['station_finder', 'monthly_summary', 'climatology', 'data_comparison']:
initial['add_degree_days'] = Get('add_degree_days', 'F')
if initial['units'] == 'metric':
initial['degree_days'] = Get('degree_days', 'gdd13,hdd21').replace(', ', ',')
else:
initial['degree_days'] = Get('degree_days', 'gdd55,hdd70').replace(', ',',')
#Set dates
#if 'grid' in initial.keys():
sd, ed, sd_fut, ed_fut = set_min_max_dates(initial)
if app_name in ['monthly_summary','climatology']:
initial['start_year'] = Get('start_year', None)
if initial['start_year'] is None:
#Link from station finder
initial['start_year'] = Get('start_date', '9999')[0:4]
if initial['start_year'] == '9999':
if 'location' in initial.keys():initial['start_year'] = sd[0:4]
else:initial['start_year'] = 'POR'
initial['end_year'] = Get('end_year', None)
if initial['end_year'] is None:
#Link from station finder
initial['end_year'] = Get('end_date', '9999')[0:4]
if initial['end_year'] == '9999':
if 'location' in initial.keys():initial['end_year'] = ed[0:4]
else:initial['end_year'] = 'POR'
initial['min_year'] = Get('min_year',sd[0:4])
initial['max_year'] = Get('max_year', ed[0:4])
initial['min_year_fut'] = sd_fut[0:4]
initial['max_year_fut'] = ed_fut[0:4]
elif app_name == 'monthly_spatial_summary':
initial['year'] = Get('year',str(int(ed[0:4]) - 1))
initial['min_year'] = Get('min_year',sd[0:4])
initial['max_year'] = Get('max_year', ed[0:4])
initial['min_year_fut'] = sd_fut[0:4]
initial['max_year_fut'] = ed_fut[0:4]
initial['season'] = Get('season','1')
elif app_name in ['seasonal_summary', 'intraannual']:
initial['start_year'] = Get('start_year','POR')
initial['end_year'] = Get('end_year','POR')
initial['start_month'] = Get('start_month', '1')
initial['start_day'] = Get('start_day', '1')
initial['min_year_fut'] = sd_fut[0:4]
initial['max_year_fut'] = ed_fut[0:4]
if app_name == 'seasonal_summary':
initial['min_year'] = Get('min_year',sd[0:4])
initial['max_year'] = Get('max_year', ed[0:4])
initial['end_month'] = Get('end_month', '1')
initial['end_day'] = Get('end_day', '31')
if app_name in ['intraannual']:
if initial['start_year'].lower() != 'por':
initial['min_year'] = initial['start_year']
else:
initial['min_year'] = Get('min_year',sd[0:4])
if initial['end_year'].lower() != 'por':
initial['max_year'] = initial['end_year']
else:
initial['max_year'] = Get('max_year', ed[0:4])
#Plotting vars
initial['show_climatology'] = Get('show_climatology','F')
initial['show_percentile_5'] = Get('show_percentile_5','F')
initial['show_percentile_10'] = Get('show_percentile_10','F')
initial['show_percentile_25'] = Get('show_percentile_25','F')
initial['target_year'] = Get('target_year_figure', None)
if initial['target_year'] is None:
initial['target_year'] = Get('target_year_form',initial['min_year'])
if initial['element'] in ['pcpn','snow','evap','pet']:
initial['calculation'] = Get('calculation','cumulative')
else:
initial['calculation'] = Get('calculation','values')
else:
initial['start_date'] = Get('start_date', WRCCUtils.format_date_string(fourtnight,'-'))
initial['end_date'] = Get('end_date', WRCCUtils.format_date_string(yesterday,'-'))
#data windows and flags
sw = '01-01'; ew = '01-31'
if 'start_date' in initial.keys() and 'end_date' in initial.keys():
if initial['start_date'] and initial['end_date']:
sw, ew = WRCCUtils.set_start_end_window(initial['start_date'],initial['end_date'])
if app_name in ['single_lister', 'multi_lister','map_overlay']:
initial['start_window'] = Get('start_window', sw)
initial['end_window'] = Get('end_window',ew)
initial['temporal_resolution'] = Get('temporal_resolution','dly')
initial['show_flags'] = Get('show_flags', 'F')
initial['show_observation_time'] = Get('show_observation_time', 'F')
if app_name in ['station_finder']:
initial['start_window'] = Get('start_window', sw)
initial['end_window'] = Get('end_window',ew)
#data summaries
if app_name in ['monthly_spatial_summary','temporal_summary', 'seasonal_summary']:
initial['data_summary'] = Get('data_summary', 'temporal_summary')
elif app_name in ['spatial_summary','multi_lister','map_overlay']:
initial['data_summary'] = Get('data_summary', 'spatial_summary')
else:
initial['data_summary'] = Get('data_summary', 'none')
if app_name in ['temporal_summary', 'monthly_spatial_summary','seasonal_summary', 'sf_download']:
if 'element' in initial.keys() and initial['element'] in ['pcpn','snow','evap','pet']:
initial['temporal_summary'] = Get('temporal_summary', 'sum')
else:
initial['temporal_summary'] = Get('temporal_summary', 'mean')
else:
initial['temporal_summary'] = Get('temporal_summary', 'mean')
if app_name in ['single_lister', 'multi_lister','spatial_summary','sf_download','map_overlay']:
initial['spatial_summary'] = Get('spatial_summary', 'mean')
#download options
if app_name in ['single_lister','multi_lister']:
initial['data_format'] = Get('data_format', 'html')
else:
initial['data_format'] = Get('data_format', 'xl')
if app_name in ['single_lister','multi_lister','station_finder','sf_download','map_overlay']:
initial['output_format'] = Get('output_format', 'verbose')
initial['delimiter'] = Get('delimiter', 'space')
initial['output_file_name'] = Get('output_file_name', 'Output')
initial['user_name'] = Get('user_name', 'Your Name')
initial['user_email'] = Get('user_email', 'Your Email')
#Set app specific params
if app_name == 'monthly_spatial_summary':
initial['area_reduce'] = Get('area_reduce','climate_division')
initial['area_statistic'] = Get('area_statistic','mean')
if app_name in ['multi_lister','spatial_summary','station_finder']:
initial['feature_id'] = 1
if app_name in ['monthly_summary','climatology','sf_link']:
initial['max_missing_days'] = Get('max_missing_days', '5')
if app_name in ['station_finder','map_overlay','sf_download']:
initial['elements_constraints'] = Get('elements_constraints', 'all')
initial['dates_constraints'] = Get('dates_constraints', 'all')
initial['display'] = Get('display', 'map')
all_meta = ['name','state','ll','elev','ids','networks','valid_daterange']
initial['metadata_keys'] = Getlist('metadata_keys',all_meta)
initial['metadata_keys_str'] = ','.join(initial['metadata_keys'])
initial['metadata_names'] = [WRCCData.DISPLAY_PARAMS[meta] for meta in initial['metadata_keys']]
initial['metadata_names_str'] = ','.join(initial['metadata_names'])
if app_name in ['monthly_summary','sf_link']:
initial['start_month'] = Get('start_month','01')
if initial['element'] in ['pcpn','snow','evap','pet']:
initial['statistic'] = Get('statistic','msum')
else:
initial['statistic'] = Get('statistic','mave')
initial['less_greater_or_between'] = Get('less_greater_or_between','b')
initial['threshold_low_for_between'] = Get('threshold_low_for_between',0.01)
initial['threshold_high_for_between'] = Get('threshold_high_for_between',0.1)
initial['threshold_for_less_than'] = Get('threshold_for_less_than',1)
initial['threshold_for_greater_than'] = Get('threshold_for_greater_than',1)
initial['departures_from_averages'] = Get('departures_from_averages','F')
initial['frequency_analysis'] = Get('frequency_analysis','F')
#Set initial plot options
initial['chart_summary'] = Get('chart_summary','individual')
#initial['plot_months'] = Get('plot_months','0,1')
if app_name == 'monthly_summary':
initial['base_temperature'] = Get('base_temperature','65')
initial['statistic_period'] = Get('statistic_period','monthly')
if app_name in ['climatology','sf_link']:
initial['summary_type'] = Get('summary_type', 'all')
if app_name == 'temporal_summary':
initial['show_plot_opts'] = Get('show_plot_opts','T')
initial['image_size'] = Get('image_size', 'medium')
initial['level_number'] = Get('level_number', '5')
initial['cmap'] = Get('cmap', 'rainbow')
initial['cmaps'] = WRCCData.CMAPS
initial['map_ol'] = Get('map_ol', 'state')
initial['interpolation'] = Get('interpolation', 'cspline')
initial['projection'] = Get('projection', 'lcc')
#Ploting options for all pages that have charts
if app_name in ['monthly_summary', 'spatial_summary','seasonal_summary', 'intraannual','data_comparison','map_overlay']:
if app_name in ['spatial_summary','monthly_summary','intraannual','map_overlay']:
if app_name in ['spatial_summary','monthly_spatial_summary','map_overlay']:
shown_indices = ','.join([str(idx) for idx in range(len(initial['elements']))])
elif app_name == 'intraannual':
shown_indices = str(int(initial['target_year']) - int(initial['min_year']))
else:
shown_indices = '0'
initial['chart_indices_string'] = Get('chart_indices_string',shown_indices)
initial['chart_type'] = Get('chart_type','spline')
initial['show_running_mean'] = Get('show_running_mean','F')
if app_name in ['monthly_summary', 'seasonal_summary']:
initial['running_mean_years'] = Get('running_mean_years',5)
else:
initial['running_mean_days'] = Get('running_mean_days',9)
initial['show_average'] = Get('show_average','F')
if app_name in ['monthly_summary']:
initial['show_range'] = Get('show_range','F')
initial['form_options'] = WRCCData.SCENIC_FORM_OPTIONS[app_name]
return initial
def sods(request, app_name):
units = {'pcpn':'Hundredths of Inches', 'snow':'Tenths of Inches', 'snwd': 'Inches', 'maxt':'Whole Degrees', 'mint':'Whole Degrees',\
'avgt':'Whole Degrees', 'dtr':'Whole Degrees', 'hdd':'Days', 'cdd':'Days','gdd':'Days'}
months = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
context = {
'title': '%s' % app_name,
}
#Form 1 dealing with station selection and for some apps, preliminary information
form1 = set_as_form(request, 'Sod0', init={'app_name': app_name})
context['form1'] = form1
if 'stn_selection' in request.POST:
#if app_name == 'Soddd':
# form1 = set_as_form(request, 'Sod0', init={'app_name': app_name, 'skip_days':False})
#elif app_name == 'Sodthr':
# form1 = set_as_form(request, 'Sod0', init={'app_name': app_name, 'custom_tables':False, 'number_of_threholds':1})
#else:
form1 = set_as_form(request, 'Sod0', init={'app_name': app_name})
context['form1'] = form1
if form1.is_valid():
context['form_2_ready'] = '2ready'
station_selection = form1.cleaned_data['station_selection']
if app_name == 'Soddd':
initial = {'app_name':app_name, 'station_selection':station_selection, \
'skip_days':form1.cleaned_data['skip_days'], 'truncate':form1.cleaned_data['truncate'] }
elif app_name == 'Sodpct':
initial = {'app_name':app_name, 'station_selection':station_selection,\
'threshold':form1.cleaned_data['threshold'], 'element':form1.cleaned_data['element'], \
'individual_averages': form1.cleaned_data['individual_averages'] }
elif app_name == 'Sodthr':
initial = {'app_name':app_name, 'station_selection':station_selection,\
'custom_tables':form1.cleaned_data['custom_tables'],'number_of_thresholds': form1.cleaned_data['number_of_thresholds']}
elif app_name == 'Sodxtrmts':
initial = {'app_name':app_name, 'station_selection':station_selection,\
'statistic':form1.cleaned_data['statistic'], 'element':form1.cleaned_data['element'], \
'frequency_analysis':form1.cleaned_data['frequency_analysis'],'statistic_period':'monthly'}
elif app_name == 'Sodpiii':
initial = {'app_name':app_name, 'station_selection':station_selection,\
'skew':form1.cleaned_data['skew'], 'cv':form1.cleaned_data['cv'], 'mean':form1.cleaned_data['mean'], \
'pct_average':form1.cleaned_data['pct_average'], 'days':form1.cleaned_data['days']}
else:
initial = {'app_name':app_name, 'station_selection':station_selection}
form2 = set_as_form2(init=initial)
context['form2'] = form2
else:
station_selection=None
#Form2 Application parameters
if 'app_form' in request.POST:
form2 = set_as_form(request, 'Sod')
context['form2'] = form2
#import pdb; pdb.set_trace()
if form2.is_valid():
context['cleaned'] = form2.cleaned_data
#Check for POR
vd = None
if 'station_id' in form2.cleaned_data.keys() or 'stnid' in form2.cleaned_data.keys():
if 'station_id' in form2.cleaned_data.keys():
stn_id = form2.cleaned_data['station_id']
if 'stnid' in form2.cleaned_data.keys():
stn_id = form2.cleaned_data['stn_id']
if 'start_date' in form2.cleaned_data.keys() and form2.cleaned_data['start_date'].upper() == 'POR':
vd = WRCCUtils.find_valid_daterange(stn_id, max_or_min='max')
if 'end_date' in form2.cleaned_data.keys() and form2.cleaned_data['end_date'].upper() == 'POR' and vd is None:
vd = WRCCUtils.find_valid_daterange(stn_id, max_or_min='max')
if vd is not None and len(vd) == 2:
form2.cleaned_data['start_date'] = vd[0]
form2.cleaned_data['end_date'] = vd[1]
(data, dates, elements, station_ids, station_names) = AcisWS.get_sod_data(form2.cleaned_data, app_name)
#get contexts for the different apps and run data application
if app_name in ['Sodrun', 'Sodrunr']:
if elements == ['maxt', 'mint']:
context['el'] = 'range'
else:
context['el'] = str(elements[0])
if form2.cleaned_data['aeb'] == 'A':
context['op'] = '>'
elif form2.cleaned_data['aeb'] == 'B':
context['op'] = '<'
else:
context['op'] = '='
context['thresh'] = form2.cleaned_data['threshold']
context['dur'] = form2.cleaned_data['minimum_run']
app_args = {'app_name': app_name, 'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names,'op':context['op'],\
'thresh':context['thresh'], 'verbose': form2.cleaned_data['verbose'], 'minimum_run': form2.cleaned_data['minimum_run']}
results = WRCCDataApps.Sodrun(**app_args)
elif app_name == 'Soddynorm':
app_args = {'app_name': app_name, 'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names,\
'filter_type':form2.cleaned_data['filter_type'],\
'filter_days':form2.cleaned_data['number_of_days']}
results = WRCCDataApps.Soddynorm(**app_args)
'''
results = run_data_app(app_name, data, dates, elements, station_ids, station_names, \
form2.cleaned_data['filter_type'], form2.cleaned_data['number_of_days'])
'''
elif app_name == 'Soddyrec':
app_args = {'app_name': app_name, 'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names}
results = WRCCDataApps.Soddyrec(**app_args)
#context['data'] =results
elif app_name == 'Soddd':
base_temp = form2.cleaned_data['base_temperature']
output_type = form2.cleaned_data['output_type']
max_miss = form2.cleaned_data['max_missing_days']
a_b = form2.cleaned_data['above_or_below']
if a_b == 'a':
context['a_b'] = 'ABOVE'
else:
context['a_b'] = 'BELOW'
max_miss = form2.cleaned_data['max_missing_days']
ncdc_round = form2.cleaned_data['ncdc_roundoff']
context['ncdc_round'] = ncdc_round
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names,\
'base_temp':base_temp, 'a_b':a_b,'output_type':output_type, \
'max_miss':max_miss, 'ncdc_round':ncdc_round}
context['skip_max_above'] = 'NO DAYS SKIPPED'
context['skip_min_below'] = 'NO DAYS SKIPPED'
context['trunc_high'] = 'NONE'
context['trunc_low'] = 'NONE'
if form2.cleaned_data['skip_days']:
skip_max_above = form2.cleaned_data['skip_days_with_max_above']
skip_min_below = form2.cleaned_data['skip_days_with_min_below']
app_args['skip_max_above'] = skip_max_above
app_args['skip_min_below'] = skip_min_below
context['skip_max_above'] = skip_max_above
context['skip_min_below'] = skip_min_below
if form2.cleaned_data['truncate']:
trunc_high = form2.cleaned_data['truncation_upper_limit']
trunc_low = form2.cleaned_data['truncation_lower_limit']
app_args['trunc_high'] = trunc_high
app_args['trunc_low'] = trunc_low
context['trunc_high'] = trunc_high
context['trunc_low'] = trunc_low
context['base_temp'] = base_temp
context['output_type'] = output_type
context['max_miss'] = max_miss
if form2.cleaned_data['output_type'] == 'm':
context['monthly'] = 'yes'
else:
context['daily'] = 'yes'
results = run_data_app(**app_args)
elif app_name == 'Sodpad':
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names}
results = run_data_app(**app_args)
mon_dict = {}
day_dict = {}
for doy in range(366):
mon, day = WRCCUtils.compute_mon_day(doy+1)
mon_dict[doy] = mon
day_dict[doy] = day
context['mon'] = mon_dict
context['day'] = day_dict
context['durations'] ={ 1:1,2:2,3:3,4:4,5:5,6:6,7:7,8:8,9:9,10:10,11:12,12:14,13:15,14:16,15:18,16:20,17:22,18:24,19:25,20:26,21:28,22:30}
elif app_name == 'Sodsumm':
el_type = form2.cleaned_data['element']
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'ids':station_ids,'station_names':station_names,'el_type':el_type, 'max_missing_days':form2.cleaned_data['max_missing_days']}
results = run_data_app(**app_args)
context['max_missing_days'] = form2.cleaned_data['max_missing_days']
if el_type == 'temp':
table_list = ['temp']
elif el_type == 'prsn':
table_list = ['prsn']
elif el_type == 'both':
table_list = ['temp', 'prsn']
elif el_type == 'hc':
table_list = ['hdd', 'cdd']
elif el_type == 'g':
table_list = ['gdd', 'corn']
elif el_type == 'all':
table_list = ['temp', 'prsn', 'hdd', 'cdd', 'gdd', 'corn']
context['headers'] = set_sodsumm_headers(table_list)
elif app_name == 'Sodpct':
el_type = form2.cleaned_data['element']
if abs(form2.cleaned_data['threshold'] + 9999) < 0.05:
threshold = None
else:
threshold = form2.cleaned_data['threshold']
ia = form2.cleaned_data['individual_averages']
number_days_ahead = form2.cleaned_data['number_days_ahead']
if 'threshold_ab' in form2.cleaned_data.keys():
threshold_ab = form2.cleaned_data['threshold_ab']
else:
threshold_ab = None
if el_type in ['hdd', 'cdd', 'gdd']:
base_temperature = form2.cleaned_data['base_temperature']
if el_type == 'gdd':
min_temperature = form2.cleaned_data['min_temperature']
max_temperature = form2.cleaned_data['max_temperature']
else:
min_temperature = None
max_temperature = None
else:
base_temperature = None
min_temperature = None
max_temperature = None
if 'begin_month' in form2.cleaned_data.keys():
begin_month = form2.cleaned_data['begin_month']
else:
begin_month = None
if 'accumulate_over_season' in form2.cleaned_data.keys():
accumulate_over_season = form2.cleaned_data['accumulate_over_season']
else:
accumulate_over_season = None
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names,'el_type':el_type,\
'ia':ia, 'number_days_ahead':number_days_ahead,'threshold':threshold, 'threshold_ab':threshold_ab, \
'base_temperature':base_temperature, 'min_temperature':min_temperature, \
'max_temperature':max_temperature, 'begin_month':begin_month, 'accumulate_over_season':accumulate_over_season }
if threshold_ab == 'A':
context['op'] = 'Above'
elif threshold_ab == 'B':
context['op'] = 'Below'
context['base_temperature'] = base_temperature
context['min_temperature'] = min_temperature
context['min_temperature'] = max_temperature
context['begin_month'] = begin_month
context['threshold'] = threshold
if threshold_ab == 'a':context['op'] = 'ABOVE'
if threshold_ab == 'b':context['op'] = 'BELOW'
context['element'] = el_type
context['units'] = units[el_type]
context['individual_averages'] = ia
results = run_data_app(**app_args)
elif app_name == 'Sodthr':
header = {}
el_type = form2.cleaned_data['element']
start_year = dates[0][0:4]
end_year = dates[-1][0:4]
number_of_thresholds = form2.cleaned_data['number_of_thresholds']
custom_tables = form2.cleaned_data['custom_tables']
if custom_tables == 'T':
time_series = []
thresholds = []
#set headers
for k in range(3):
header[k] = set_sodthr_headers(k, el_type, str(form2.cleaned_data['interval_start']), \
str(form2.cleaned_data['midpoint']),str(form2.cleaned_data['interval_end']), start_year, end_year, \
int(form2.cleaned_data['max_missing_days_first_and_last']), int(form2.cleaned_data['max_missing_days_differences']), \
str(form2.cleaned_data['above_or_below']), str(form2.cleaned_data['latest_or_earliest_for_period_1']), \
str(form2.cleaned_data['latest_or_earliest_for_period_2']))
#Find list of thresholds, and time_series_booleans
for k in range(int(number_of_thresholds)):
thresholds.append(float(form2.cleaned_data['threshold_%s' % str(k)]))
time_series.append(form2.cleaned_data['time_series_%s' % str(k)])
#set application arguments
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names,'el_type':el_type, 'custom_tables':True, \
'interval_start':form2.cleaned_data['interval_start'], 'midpoint':form2.cleaned_data['midpoint'], \
'interval_end':form2.cleaned_data['interval_end'], 'thresholds': thresholds, 'time_series': time_series, \
'le_1': form2.cleaned_data['latest_or_earliest_for_period_1'], \
'le_2':form2.cleaned_data['latest_or_earliest_for_period_2'], 'ab':form2.cleaned_data['above_or_below'], \
'miss_days_1':form2.cleaned_data['max_missing_days_first_and_last'], 'miss_days_2':form2.cleaned_data['max_missing_days_differences']}
else:
#set headers
for k in range(3):
header[k] = set_sodthr_headers(k, el_type, '0101', '0731', '1231', start_year, \
end_year, 10, 10, 'BELOW', 'latest','earliest')
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names,'el_type':el_type, 'custom_tables':False}
context['header'] = header
results = run_data_app(**app_args)
elif app_name == 'Sodxtrmts':
context['element'] = form2.cleaned_data['element']
context['max_missing_days'] = form2.cleaned_data['max_missing_days']
context['start_month'] = WRCCData.NUMBER_TO_MONTH_NAME[form2.cleaned_data['start_month']]
mon_start = int(form2.cleaned_data['start_month'].lstrip('0'))
month_list = []
for mon_idx in range(mon_start -1,12):
month_list.append(months[mon_idx])
for mon_idx in range(mon_start-1):
month_list.append(months[mon_idx])
month_list.append('ANN')
context['month_list'] = month_list
context['frequency_analysis'] = WRCCData.DISPLAY_PARAMS[form2.cleaned_data['frequency_analysis']]
context['statistic'] = WRCCData.DISPLAY_PARAMS[form2.cleaned_data['statistic']]
context['departure_from_averages'] = WRCCData.DISPLAY_PARAMS[form2.cleaned_data['departures_from_averages']]
app_args = {
'app_name':app_name,
'data':data,
'dates':dates,
'elements':elements,
'station_ids':station_ids,
'station_names':station_names,
'element':form2.cleaned_data['element'],
'max_missing_days':form2.cleaned_data['max_missing_days'],
'start_month': form2.cleaned_data['start_month'],
'statistic_period':'monthly',
'statistic': form2.cleaned_data['statistic'],
'frequency_analysis': form2.cleaned_data['frequency_analysis'],
'departures_from_averages':form2.cleaned_data['departures_from_averages']
}
if form2.cleaned_data['frequency_analysis'] == 'T':
context['frequency_analysis_type'] = form2.cleaned_data['frequency_analysis_type']
app_args['frequency_analysis_type'] = form2.cleaned_data['frequency_analysis_type']
if form2.cleaned_data['statistic'] == 'ndays':
app_args['less_greater_or_between'] = form2.cleaned_data['less_greater_or_between']
if form2.cleaned_data['less_greater_or_between'] == 'b':
app_args['threshold_low_for_between'] = form2.cleaned_data['threshold_low_for_between']
app_args['threshold_high_for_between'] = form2.cleaned_data['threshold_high_for_between']
else:
app_args['threshold_for_less_or_greater'] = form2.cleaned_data['threshold_for_less_or_greater']
if form2.cleaned_data['element'] in ['hdd', 'gdd', 'cdd']:
app_args['base_temperature'] = form2.cleaned_data['base_temperature']
context['app_arg'] = app_args
results = WRCCDataApps.Sodxtrmts(**app_args)
#context['fa_results'] = dict(fa_results)
elif app_name == 'Sodpiii':
lisdur = [ '6 Hrs', '12 Hrs', '1 Day', '2 Days', '3 Days', \
'4 Days', '5 Days', '6 Days', '7 Days', '8 Days', '9 Days', \
'10 Days', '15 Days', '20 Days', '25 Days', '30 Days']
context['el_type'] = form2.cleaned_data['element']
context['units'] = units[form2.cleaned_data['element']]
context['start_year'] = form2.cleaned_data['start_date'][0:4]
context['end_year'] = str(int(form2.cleaned_data['end_date'][0:4]) - 1)
context['start_month'] = form2.cleaned_data['start_date'][4:6]
context['end_month'] = form2.cleaned_data['end_date'][4:6]
app_args = {'app_name':app_name,'data':data,'dates':dates,'elements':elements,\
'station_ids':station_ids,'station_names':station_names, \
'el_type':form2.cleaned_data['element'], 'skew':form2.cleaned_data['skew'], \
'cv':form2.cleaned_data['cv'], 'mean': form2.cleaned_data['mean'], \
'pct_average':form2.cleaned_data['pct_average'], \
'value_subsequent':form2.cleaned_data['value_subsequent'], \
'value_missing':form2.cleaned_data['value_missing'],'days':form2.cleaned_data['days']}
if form2.cleaned_data['element'] == 'avgt':
app_args['ab'] = form2.cleaned_data['mean_temperatures']
duration = {}
if form2.cleaned_data['days'] == 'i':
app_args['number_of_days'] = int(form2.cleaned_data['number_of_days'])
duration[0] = '%i Days' % int(form2.cleaned_data['number_of_days'])
elif form2.cleaned_data['days'] == '5':
for k in range(5):
duration[k] = lisdur[k + 2]
elif form2.cleaned_data['days'] == 'a':
for k in range(len(lisdur)):
duration[k] = lisdur[k]
context['duration'] = duration
results_0, results, averages, stdevs, skews = WRCCDataApps.Sodpiii(**app_args)
context['averages'] = averages
context['stdevs'] = stdevs
context['skews'] = skews
context['results_0'] = dict(results_0)
else:
results = {}
#general context
try:
context['results'] = dict(results)
except:
context['results'] = results
context['dates'] = dates
if app_name in ['Sodrun', 'Sodrunr', 'Soddyrec', 'Sodcnv', 'Sodlist']:
context['start_year'] = dates[0]
context['end_year'] = dates[-1]
elif app_name in ['Soddd', 'Sodpiii', 'Sodsumm']:
context['start_year'] = dates[0][0:4]
context['end_year'] = str(int(dates[-1][0:4]))
else:
context['start_year'] = dates[0][0:4]
if app_name == 'Sodxtrmts':
context['end_year'] = dates[-1][0:4]
else:
context['end_year'] = str(int(dates[-1][0:4])+1)
context['num_yrs'] = int(dates[-1][0:4]) - int(dates[0][0:4])+1
context['elements'] = dict([(k,v) for k,v in enumerate(elements)])
context['data'] = dict(data)
context['station_ids'] = dict([(k,v) for k,v in enumerate(station_ids)])
context['station_names'] = dict([(k,v) for k,v in enumerate(station_names)])
#form_2 not valid or we are done with analysis
form2 = set_as_form(request, 'Sod')
context['form2'] = form2
context['form_2_ready'] = True
station_selection = None
#else: #form_2 is not valid
# print "The following errors occurred:"
# print form2.errors
# station_selection = None
return render_to_response('wrcc_apps/%s.html' % app_name, context, context_instance=RequestContext(request))
def set_form(request, clean=True):
'''
Coverts request input to usable form input:
Deals with unicode issues
and autofill options for identifiers
NOTE: elements should always be a list (also when clean = False)
If Clean == True,
We also clean up some form fields for submission:
date fields, convert to yyyymmdd
window fields, convert to mmdd
name strings are converted to ids
Combine elemenst weith degree days
'''
try:
req_method = request.method
except:
if isinstance(request,dict):
req_method = 'dict'
else:req_method = None
form= {}
form['req_method'] = req_method
#Convert request object to python dictionary
if req_method == 'dict':
form = copy.deepcopy(request)
#Special case elements, always needs to be list
if 'element' in request.keys() and not 'elements' in request.keys():
form['elements'] = [form['element']]
if 'elements' in request.keys():
form['elements'] = WRCCUtils.convert_elements_to_list(request['elements'])
elif req_method == 'POST':
for key, val in request.POST.items():
form[str(key)]= val
#form = dict((str(x),str(y)) for x,y in request.POST.items())
#Special case elements, always needs to be list
if 'element' in request.POST.keys() and not 'elements' in request.POST.keys():
form['elements'] = [str(request.POST['element'])]
if 'elements' in request.POST.keys():
#form['elements'] = WRCCUtils.convert_elements_to_list(request.POST['elements'])
els = request.POST.getlist('elements',request.POST.get('elements','').split(','))
form['elements'] = [str(el) for el in els]
if 'metadata_keys' in request.POST.keys():
form['metadata_keys'] = request.POST.getlist('metadata_keys',request.POST.get('metadata_keys','').split(','))
elif req_method == 'GET':
#form = dict((str(x),str(y)) for x,y in request.GET.items())
for key, val in request.GET.items():
form[str(key)]= val
#Special case elements, always needs to be list
if 'element' in request.GET.keys() and not 'elements' in request.GET.keys():
form['elements'] = [str(request.GET['element'])]
if 'elements' in request.GET.keys():
#form['elements'] = WRCCUtils.convert_elements_to_list(request.GET['elements'])
form['elements'] = request.GET.get('elements','').split(',')
if 'metadata_keys' in request.GET.keys():
form['metadata_keys'] = request.GET.getlist('metadata_keys',request.GET.get('metadata_keys','').split(','))
else:
form = {}
#set data type for single apps
if 'data_type' not in form.keys():
if 'station_id' in form.keys():
form['data_type'] = 'station'
if 'location' in form.keys():
form['data_type'] = 'grid'
if 'app_name' in form.keys() and form['app_name'] in ['temporal_summary','monthly_spatial_summary']:
form['data_type'] = 'grid'
#Convert unicode to string
if 'elements' in form.keys():
form['elements'] = [str(el) for el in form['elements']]
if 'csrfmiddlewaretoken' in form.keys():
del form['csrfmiddlewaretoken']
if 'formData' in form.keys():
del form['formData']
if 'form_options' in form.keys():
del form['form_options']
if not clean:
return form
#Clean up form for submission
#Clean Dates and windows
vd = None
for key in ['start_date', 'end_date', 'start_year', 'end_year','start_window','end_window']:
if key not in form.keys():
continue
if form[key].lower() == 'por':
if str(key) in ['start_date']:
k=key; idx = 0;sd = 'por'; ed = form['end_date']
if str(key) in ['end_date']:
k=key; idx = 1;ed = 'por'; sd = form['start_date']
if str(key) in ['start_year']:
k='start_date'; idx = 0;sd = 'por'
if form['end_year'].lower() == 'por':ed = 'por'
else:ed = str(int(form['end_year']) -1) + '-12-31'
if str(key) in ['end_year']:
k='end_date'; idx = 1;ed = 'por'
if form['start_year'].lower() == 'por':sd = 'por'
else:sd = form['start_year'] + '-01-01'
if 'element' in form.keys() and not 'elements' in form.keys():
if form['element'] in ['dtr']:
el_list = ['maxt','mint']
if form['element'] in ['pet']:
el_list = ['maxt','mint','pcpn']
if 'elements' in form.keys() and not 'element' in form.keys():
if isinstance(form['elements'],basestring):
el_list = form['elements'].replace(' ','').split(',')
else:
el_list = form['elements']
else:
el_list = None
if 'station_id' in form.keys():
if vd is None:
stn_id, stn_name = WRCCUtils.find_id_and_name(str(form['station_id']),settings.MEDIA_DIR +'json/US_station_id.json')
vd = WRCCUtils.find_valid_daterange(stn_id, start_date=sd, end_date=ed, el_list=el_list, max_or_min='max')
form[k] = vd[idx]
if key == 'start_year' and form['start_year'].lower() == 'por':
form['start_year'] = vd[0][0:4]
if key == 'end_year' and form['end_year'].lower() == 'por':
form['end_year'] = vd[1][0:4]
else:
form[str(key)] = str(form[key]).replace('-','').replace(':','').replace('/','').replace(' ','')
else:
form[str(key)] = str(form[key]).replace('-','').replace(':','').replace('/','').replace(' ','')
#Convert user input of area names to ids
for key in ['station_id','county', 'basin', 'county_warning_area', 'climate_division']:
if not key in form.keys():
continue
ID,name = WRCCUtils.find_id_and_name(form[key],settings.MEDIA_DIR +'json/US_' + key + '.json')
form[key] = ID
form['user_area_id'] = str(name) + ', ' + str(ID)
if not 'user_area_id' in form.keys():
try:
form['user_area_id'] = form[form['area_type']]
except:
try:
form['user_area_id'] = form[form['data_type']]
except:
pass
#station_ids is special case
if 'station_ids' in form.keys():
stn_ids = ''
stn_list = form['station_ids'].rstrip(',').split(',')
#Remove leading spaces from list items
stn_list = [v.lstrip(' ').rstrip(' ') for v in stn_list]
stn_ids, stn_names = WRCCUtils.find_ids_and_names(stn_list,settings.MEDIA_DIR +'json/US_' + 'station_id' + '.json')
form['station_ids'] = stn_ids
uai = ''
stn_names_list = stn_names.split(',')
for idx, stn_id in enumerate(stn_ids.split(',')):
uai+=str(stn_names[idx]) + ', ' + str(stn_id) + ';'
form['user_area_id'] = uai
#set data summary if needed
if 'data_summary' not in form.keys():
if 'temporal_summary' in form.keys():
form['data_summary'] = 'temporal_summary'
if 'spatial_summary' in form.keys():
form['data_summary'] = 'spatial_summary'
#Combine elements
if 'add_degree_days' in form.keys() and form['add_degree_days'] == 'T':
for dd in form['degree_days'].replace(' ','').split(','):
'''
if form['units'] == 'metric':
el_strip, base_temp = WRCCUtils.get_el_and_base_temp(dd)
form['elements'].append(el_strip + str(WRCCUtils.convert_to_english('base_temp',base_temp)))
else:
form['elements'].append(dd)
'''
form['elements'].append(dd)
return form
from django.conf import settings
import WRCCUtils, WRCCData, AcisWS
import copy
today = WRCCUtils.set_back_date(0)
today_year = today[0:4]
today_month = today[5:7]
today_day = today[8:10]
begin_10yr = WRCCUtils.set_back_date(3660)
yesterday = WRCCUtils.set_back_date(1)
fourtnight = WRCCUtils.set_back_date(14)
def create_kml_file(area_type, overlay_state):
kml_file_name = overlay_state + '_' + area_type + '.kml'
kml_file_path = settings.TMP_URL + kml_file_name
status = WRCCUtils.generate_kml_file(area_type, overlay_state, kml_file_name, settings.TEMP_DIR)
if status != 'Success':
return 'ERROR: ' + status
return kml_file_path
def set_GET(request):
try:
rm = request.method
except:
def Get(key, default):
if key in request.keys():
return request[key]
else:
return default