def prepro_window(prepro_window_option=None): """ Returns the preprocessing window corresponding to the run The prepro_window_option can be set to "RUNDATE" "16yr" or "82yr" and the prepro time window will be buffered to the nearest day, the standard 16yr preprocessing window or 82 years accordingly. The default is RUNDATE. """ if prepro_window_option == None: prepro_window_option == getAttr("PREPRO_WINDOW") if not prepro_window_option: prepro_window_option = "RUNDATE" if (prepro_window_option == "RUNDATE"): st = getAttr("START_TIME") if st != "0000" and st != "2400": st = "0000" et = getAttr("END_TIME") if et != "2400" and et != "0000": et = "2400" return timewindow( getAttr("START_DATE") + " " + st + " - " + getAttr("END_DATE") + " " + et) elif prepro_window_option.lower() == "16yr": return timewindow("01OCT1974 0000 - 01OCT1991 0000") elif prepro_window_option.lower() == "82yr": return timewindow("01OCT1921 0000 - 01OCT2003 0000") else: raise "Prepro time window option not understood: %s" % prepro_window_option
def prepro_window(prepro_window_option = None): """ Returns the preprocessing window corresponding to the run The prepro_window_option can be set to "RUNDATE" "16yr" or "82yr" and the prepro time window will be buffered to the nearest day, the standard 16yr preprocessing window or 82 years accordingly. The default is RUNDATE. """ if prepro_window_option == None: prepro_window_option == getAttr("PREPRO_WINDOW") if not prepro_window_option: prepro_window_option = "RUNDATE" if (prepro_window_option == "RUNDATE"): st = getAttr("START_TIME") if st != "0000" and st != "2400": st = "0000" et = getAttr("END_TIME") if et != "2400" and et != "0000": et = "2400" return timewindow(getAttr("START_DATE") + " " + st + " - " + getAttr("END_DATE") + " " + et) elif prepro_window_option.lower() == "16yr": return timewindow("01OCT1974 0000 - 01OCT1991 0000") elif prepro_window_option.lower() == "82yr": return timewindow("01OCT1921 0000 - 01OCT2003 0000") else: raise "Prepro time window option not understood: %s" % prepro_window_option
def extend_flow(nodes_to_extend): """ Copying WY1922 data to WY1921 for allowing to preprocessing and running DSM2 from 01Jan1921. """ calsimfile = getAttr("CALSIMFILE") f = opendss(calsimfile) # open CALSIM file outfile = getAttr("BOUNDARYFILE") if not outfile or outfile == "": raise "Config variable BOUNDARYFILE not set and needed for prepro output" tw = timewindow("01OCT1921 0000 - 01OCT1922 0000") for calsimname in nodes_to_extend: print calsimname dsspath = calsim_path(calsimname) paths = findpath(f, dsspath) if not paths or len(paths) > 1: print "File: %s" % calsimfile raise "Path %s not found or not unique" % dsspath ref = DataReference.create(paths[0], tw) monthly = ref.getData() itr = monthly.getIterator() d = zeros(len(monthly), 'd') count = 0 while not itr.atEnd(): el = itr.getElement() d[count] = el.getY() count = count + 1 itr.advance() stime = "01OCT1920 0000" rts = RegularTimeSeries(monthly.getName(),stime, \ timeinterval("1MON").toString(), d, None, monthly.getAttributes()) writedss(calsimfile, ref.getPathname().toString(), rts)
def extend_flow(nodes_to_extend): """ Copying WY1922 data to WY1921 for allowing to preprocessing and running DSM2 from 01Jan1921. """ calsimfile=getAttr("CALSIMFILE") f=opendss(calsimfile) # open CALSIM file outfile=getAttr("BOUNDARYFILE") if not outfile or outfile == "": raise "Config variable BOUNDARYFILE not set and needed for prepro output" tw=timewindow("01OCT1921 0000 - 01OCT1922 0000") for calsimname in nodes_to_extend: print calsimname dsspath = calsim_path(calsimname) paths = findpath(f,dsspath) if not paths or len(paths)>1: print "File: %s" % calsimfile raise "Path %s not found or not unique" % dsspath ref=DataReference.create(paths[0],tw) monthly=ref.getData() itr = monthly.getIterator() d=zeros(len(monthly),'d') count=0 while not itr.atEnd(): el = itr.getElement() d[count] = el.getY() count = count + 1 itr.advance() stime = "01OCT1920 0000" rts = RegularTimeSeries(monthly.getName(),stime, \ timeinterval("1MON").toString(), d, None, monthly.getAttributes()) writedss(calsimfile,ref.getPathname().toString(),rts)
def planning_window(): """ Returns the runtime represented by START_DATE START_TIME END_DATE and END_TIME in the config file as a timewindow """ return timewindow(getAttr("START_DATE") + " " + getAttr("START_TIME") + " - " + + getAttr("END_DATE") + " " + getAttr("END_TIME"))
def planning_window(): """ Returns the runtime represented by START_DATE START_TIME END_DATE and END_TIME in the config file as a timewindow """ return timewindow( getAttr("START_DATE") + " " + getAttr("START_TIME") + " - " + +getAttr("END_DATE") + " " + getAttr("END_TIME"))
def write_water_balance_table(fh, dss_group1,dss_group2,scalars,pathname_maps,tw_values): import vtimeseries print >> fh, "<h1>System Water Balance Comparison: %s vs %s</h1>"%(scalars['NAME1'], scalars['NAME2']) print >> fh, '<div id="note">Note: %s</div>'%(scalars['NOTE'].replace('"','')) print >> fh, '<div id="assumptions">Assumptions: %s</div>'%(scalars['ASSUMPTIONS'].replace('"','')) time_windows = map(lambda val: val[1].replace('"',''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) print >> fh, '<table id="system-water-balance-table" class="alt-highlight">' print >> fh, '<caption>System Water Balance Table</caption>' print >> fh, '<tr><td colspan="4"></td>' for i in range(len(tws)): print >> fh, '<td colspan="4" class="timewindow">%s</td>'%tw_values[i][0].replace('"','') print >> fh, "</tr>" print >> fh, '<tr><td colspan="4"></td>' for tw in tws: print >> fh, '<td colspan="4" class="timewindow">%s</td>'%format_timewindow(tw) print >> fh, "</tr>" print >> fh, "<tr>" print >> fh, '<tr class="header"><td colspan="4">Output Name</td>' for tw in tws: print >> fh, '<td>%s</td><td>%s</td><td>Diff</td><td>%% Diff</td>'%((scalars['NAME2']+"<br/>Alt", scalars['NAME1']+"<br/>Base")) print >> fh, "</tr>" index=0 for path_map in pathname_maps: var_name = path_map.var_name calculate_dts=0 if path_map.report_type == 'Exceedance_Post': calculate_dts=1 ref1 = get_ref(dss_group1, path_map.path1,calculate_dts, pathname_maps, 1) ref2 = get_ref(dss_group2, path_map.path2,calculate_dts, pathname_maps, 2) if path_map.var_category in ("RF", "DI", "DO", "DE", "SWPSOD", "CVPSOD"): print >> fh, '<tr class="%s">'%("d"+str(index%2)) if (ref1==None or ref2==None): logging.debug("No data for %s"%path_map.var_name) if ref1 != None: data1=cfs2taf(ref1.data) else: data1=None logging.debug("No data for %s: %s in %s"%path_map.var_name, path_map.path1, path_map.dss_group1) if ref2 != None: data2=cfs2taf(ref2.data) else: data2=None logging.debug("No data for %s: %s in %s"%path_map.var_name, path_map.path2, path_map.dss_group2) print >> fh, '<td colspan="4">%s</td>'%(path_map.var_name.replace('"','')) for tw in tws: s1=avg(data1, tw) s2=avg(data2, tw) diff=s2-s1 if s1!=0: pct_diff=diff/s1*100.0 else: pct_diff=float('nan') print >> fh, "<td>%0.1f</td><td>%0.1f</td><td>%0.1f</td><td>%0.1f</td>"%(s2,s1,diff,pct_diff) print >> fh, "</tr>" index=index+1 print >> fh, '</table>'
def prep_ndo(calsimf, dss_step, dss_fpart, twstr): ndofile = opendss(calsimf) TWIND = timewindow(twstr) ndo = DataReference.create( findpath(ndofile, "/CALSIM/NDO/FLOW-NDO//" + dss_step + "/" + dss_fpart + "/")[0], TWIND).getData() ndo15 = conserve.conserveSpline(ndo, "15MIN") ndo15.getAttributes().setYUnits("CFS") ndo15.getAttributes().setYType("PER-AVER") writedss(calsimf, "/CALSIM/NDO/FLOW-NDO//15MIN/" + dss_fpart + "/", ndo15) return 0
def write_control_panel(fh, scalars, pathname_maps, tw_values): import vtimeseries print >> fh, """<div id="control-panel"> <select name="tw" id="time-window-select"> """ for i in range(len(tw_values)): if i==0: selected = 'selected="selected"' else: selected = "" print >> fh, '<option value="%s" %s>%s</option>'%(timewindow_option_value(vtimeseries.timewindow(tw_values[i][1].replace('"',''))), selected,tw_values[i][0].replace('"','')) print >> fh, """
def dss_retrieve_ts(file, path, *arg): f = opendss(file) g = findpath(f, path) if g == None or len(g) != 1: raise ValueError("Path %s in file s% does not exist or is not unique" % (path, file)) if len(arg) > 1: raise ValueError("usage: file, dss_path, time window") elif len(arg) == 1: twind = timewindow(arg[0]) return DataReference.create(g[0], twind).getData() else: return g[0].getData()
import sys
def do_processing(scalars, pathname_maps, tw_values): # open files 1 and file 2 and loop over to plot dss_group1 = vutils.opendss(scalars['FILE1']) dss_group2 = vutils.opendss(scalars['FILE2']) time_windows = map(lambda val: val[1].replace('"',''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) if len(tws) > 0: tw=tws[0] else: tw=None output_file=scalars['OUTFILE'] data_output_file = output_file.split(".")[0]+".js" fh=open(data_output_file,'w') print >> fh, """/* Comparison Output File Generated on : %s */"""%(str(Date())) js_data.write_begin_data_array(fh); if dss_group1 == None or dss_group2 == None: sys.exit(2); dataIndex=0 for path_map in pathname_maps: dataIndex=dataIndex+1 logging.debug('Working on index: %d'%dataIndex) if dataIndex>1: fh.write(",") #path_map = pathname_mapping[var_name] if path_map.path2==None or path_map.path2 == "": path_map.path2=path_map.path1 var_name = path_map.var_name calculate_dts=0 if path_map.var_category == 'HEADER': logging.debug('Inserting header') continue; if path_map.report_type == 'Exceedance_Post': calculate_dts=1 ref1 = get_ref(dss_group1, path_map.path1,calculate_dts, pathname_maps, 1) ref2 = get_ref(dss_group2, path_map.path2,calculate_dts, pathname_maps, 2) if (ref1==None or ref2==None): continue series_name = [scalars['NAME1'],scalars['NAME2']] data_units=get_units(ref1,ref2) data_type=get_type(ref1,ref2) if path_map.report_type == 'Average': write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES) elif path_map.report_type == 'Exceedance': write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE) elif path_map.report_type == 'Avg_Excd': write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES) fh.write(",") write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE) elif path_map.report_type == 'Timeseries': write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES) elif path_map.report_type == 'Exceedance_Post': write_plot_data(fh, build_exceedance_array(ref1,ref2,tw), dataIndex, "Exceedance %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE) js_data.write_end_data_array(fh); logging.debug('Writing end of data array') fh.close() # Generate the main html file fh=open(scalars['OUTFILE'],'w') print >> fh, """ <html> <head> <title>Calsim Report: %s vs %s</title> <script type="text/javascript" src="%s"></script> <script type="text/javascript" src="report-common/protovis-d3.3.js"></script> <script type="text/javascript" src="report-common/plots.js"></script> <script type="text/javascript" src="report-common/jquery-1.4.2.min.js"></script> <link rel="stylesheet" type="text/css" media="print" href="report-common/print.css" /> <link rel="stylesheet" type="text/css" media="screen" href="report-common/screen.css" /> </head> """%(scalars['NAME1'],scalars['NAME2'],data_output_file) write_control_panel(fh,scalars, pathname_maps,tw_values) write_water_balance_table(fh,dss_group1, dss_group2, scalars, pathname_maps,tw_values) print >> fh, """<script type="text/javascript"> function clear_and_draw(sdate, edate){ $('.plot').empty(); n=data.length plot_diff = $('input[name=diff_plot]').is(':checked') ? 1 : 0 ; for(i=0; i < n; i++){ var div_id = "fig"+(i+1); if (data[i]==null) continue; if ($("#"+div_id).length==0){ $("body").append('<a href="#'+div_id+'"><div class="plot" id="'+div_id+'"></div></a>'); } if (data[i].plot_type=="timeseries"){ plots.time_series_plot(div_id,data[i],plot_diff,sdate,edate); }else if (data[i].plot_type=="exceedance"){ plots.exceedance_plot(div_id,data[i],null,sdate,edate); } } // update links }; $(document).ready(clear_and_draw(null,null)); $('#system-water-balance-table tr').click(function(){ var_name = $($(this).find('td')[0]).text(); var svg_element = $('div').filter(function(){ return $(this).text().indexOf(var_name)>=0;}) if (svg_element && svg_element.length > 0){ anchor_name = $($(svg_element[0]).parent()).attr("href"); window.location.href=window.location.href.split('#')[0]+anchor_name; } }); </script> <script type="text/javascript"> function extract_date(date_str){ date_fields=date_str.split(","); return new Date(date_fields[0],date_fields[1],date_fields[2]); } $('#time-window-select').change(function(){ var changed_val = $('#time-window-select option:selected').val().split("-"); clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1])); }); $('#threshold').change(function(){ set_diff_threshold($('#threshold').val()); }); $('input[name=diff_plot]').change(function(){ var changed_val = $('#time-window-select option:selected').val().split("-"); clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1])); }); function set_diff_threshold(threshold){ var diff_marker = function(){ if(Math.abs(parseFloat($(this).text())) >= threshold){ $(this).addClass('large-diff'); }else{ $(this).removeClass('large-diff'); }; }; $('tr td:nth-child(5)').each(diff_marker); $('tr td:nth-child(9)').each(diff_marker); $('tr td:nth-child(13)').each(diff_marker); }; set_diff_threshold($('#threshold').val()); </script> </body> </html>""" fh.close() logging.debug('Closed out data file')
def ECEst(stage, ndo, so, sb, beta, npow1, npow2, g0, zrms, c): """ Estimate 15min EC at the boundary. Inputs: stage astronomical tide estimate. Only 15min data are acceptable ndo ndo estimate -- e.g., from CALSIM """ import interpolate from vista.set import Units if not isinstance(stage,RegularTimeSeries) or \ not isinstance(ndo,RegularTimeSeries): raise "stage and ndo must be RegularTimeSeries" if ndo.getTimeInterval().toString() == "1DAY": ndo = interpolate.spline(ndo, "15MIN", 0.5) << 95 elif ndo.getTimeInterval().toString() != "15MIN": raise "ndo must be a one day or 15 minute series" if not stage.getTimeInterval().toString() == "15MIN": raise "stage must be an hourly or 15 minute series" # if ndo.getTimeInterval().toString() != stage.getTimeInterval().toString(): raise "stage and ndo must have the same window" # #if not len(c) ==9: # raise "Wrong number (%s) of coefficients in the array c" % len(c) if (first_missing(ndo)) >= 0: raise "missing data not allowed in ndo. First missing data at index: %s" % first_missing( ndo) if (first_missing(stage)) >= 0: raise "missing data not allowed in stage. First missing data at index: %s" % first_missing( stage) newstart = ndo.getStartTime() - "21HOURS" newend = ndo.getEndTime() - "3HOURS" if (stage.getStartTime().getTimeInMinutes() - newstart.getTimeInMinutes() > 0): print "Stage record starts %s and NDO starts %s" % ( stage.getStartTime().toString(), ndo.getStartTime().toString()) raise "stage record must begin at least 21 hours before ndo" if (newend.getTimeInMinutes() - stage.getEndTime().getTimeInMinutes() > 0): raise "stage record must end no more than 3 hours before end of ndo" ztw = timewindow(newstart.toString() + ' - ' + newend.toString()) z = stage.createSlice(ztw) g = gCalc(ndo, beta, g0) # writedss("planning_ec_input.dss","/CALC/RSAC054/G//15MIN/CALC",g) # for debug zarr = z.getYArray() giter = g.getIterator() ec = zeros(g.size(), 'd') ec = map(lambda x: -901.0, ec) zrmsiter = zrms.getIterator() i = 0 while (not giter.atEnd()): gval = giter.getElement().getY() zrmsval = zrmsiter.getElement().getY() ecfrac = gval * c[0] + 1.1 * gval**npow1 * ( c[1] * zarr[i + 72] + c[2] * zarr[i + 60] + c[3] * zarr[i + 48] + c[4] * zarr[i + 36] + c[5] * zarr[i + 24] + c[6] * zarr[i + 12] + c[7] * zarr[i]) ec[i] = max(200, exp(ecfrac) * (so - sb) + sb) zrmsiter.advance() giter.advance() i = i + 1 # ec needs to be set umhos/cm rts = RegularTimeSeries("/ECest//////", g.getStartTime().toString(), g.getTimeInterval().toString(), ec) rts.getAttributes().setYUnits(Units.UMHOS_CM) return [rts, gval]
def do_processing(globals, scalars, var_values, output_values, tw_values): compare_mode = globals['COMPARE_MODE'] time_interval = globals['DEFAULT_TIME_INTERVAL'] # open files 1 and file 2 and loop over to plot from java.util import Date dss_group0, dss_group1, dss_group2, refvar, refname = compare_dss_utils.get_group_ref(globals, scalars, var_values) # obtain the new output list modified for *_ or _* cases output_values = compare_dss_utils.get_ref_list(compare_mode,dss_group0,dss_group1,output_values) output_dir = scalars['OUTDIR'].replace('"','') wyt_arr = write_wyt(output_dir, tw_values) #write the water type JavaScript file #out_name = column(output_values,0) #out_type = column(output_values,1) if compare_mode=='1': type_arr = compare_dss_utils.get_cpart_list(dss_group0) else: type_arr = compare_dss_utils.get_cpart_list(dss_group1) compare_dss_utils.chk_cpart_space(type_arr) if output_dir[-1]!='/': output_dir=output_dir+"/" time_windows = map(lambda val: val[1].replace('"',''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) dIndex = 0 dataIndex = {} fs_davg = {} fs_dmax = {} fs_dmin = {} fs_mavg = {} fs_orig = {} tbl_latlng = compare_dss_utils.lines2table(output_dir+"js/latlng.txt") fl = open(output_dir+"data/data_list.js",'w') for i in type_arr: initial_js = output_dir+"data/DSS_compare_spec_davg_"+i+".js" initial_pretab = i fs_davg[i] = open(output_dir+"data/DSS_compare_spec_davg_"+i+".js",'w') fs_dmax[i] = open(output_dir+"data/DSS_compare_spec_dmax_"+i+".js",'w') fs_dmin[i] = open(output_dir+"data/DSS_compare_spec_dmin_"+i+".js",'w') fs_mavg[i] = open(output_dir+"data/DSS_compare_spec_mavg_"+i+".js",'w') print >> fs_davg[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) print >> fs_dmax[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) print >> fs_dmin[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) print >> fs_mavg[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) dataIndex['spec_davg_'+i] = 0 dataIndex['spec_dmax_'+i] = 0 dataIndex['spec_dmin_'+i] = 0 dataIndex['spec_mavg_'+i] = 0 js_data.write_begin_data_array(fs_davg[i]) js_data.write_begin_data_array(fs_dmax[i]) js_data.write_begin_data_array(fs_dmin[i]) js_data.write_begin_data_array(fs_mavg[i]) if globals['PLOT_ORIGINAL_TIME_INTERVAL']=='ON': fs_orig[i] = open(output_dir+"data/DSS_compare_spec_orig_"+i+".js",'w') print >> fs_orig[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) dataIndex['spec_orig_'+i] = 0 js_data.write_begin_data_array(fs_orig[i]) js_data_list.write_begin_data_array(fl) wy_types = ['W','AN','BN','D','C'] if globals['DONOT_SORT_STATION_NAME']=='OFF': output_values = compare_dss_utils.sort_alphabetically(output_values) refname = compare_dss_utils.sort_alphabetically(refname) else: refname = compare_dss_utils.get_not_sort_refname(refname,output_values) for name in refname: ref0 = None; ref1 = None; ref2 = None try: series_name = compare_dss_utils.get_series_name(compare_mode,scalars,refvar[name]) except Exception,e: logging.error("Error building name for :%s and error is %s"%(name,e)) continue if (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='ON' and name in output_values): conti = 0 str_refvar = compare_dss_utils.get_str_refvar(refvar[name],compare_mode) if compare_mode=='1' or compare_mode=='4' or compare_mode=='5': try: if refvar[name][0][4]=='0': dss_group = dss_group0 if refvar[name][0][4]=='1': dss_group = dss_group1 if refvar[name][0][4]=='2': dss_group = dss_group2 ref0 = compare_dss_utils.get_ref(dss_group, refvar[name][0]) p = ref0.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref(ref0) except: print "**** Can't find matched time series for "+ name +" ****" conti = 1 if compare_mode=='3' or compare_mode=='5': try: if refvar[name][2][4]=='0': dss_group = dss_group0 if refvar[name][2][4]=='1': dss_group = dss_group1 if refvar[name][2][4]=='2': dss_group = dss_group2 ref2 = compare_dss_utils.get_ref(dss_group, refvar[name][2]) p = ref2.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref(ref2) except: print "**** Can't find matched time series for "+ name +" ****" conti = 1 if compare_mode=='2' or compare_mode=='3' or compare_mode=='4' or compare_mode=='5': try: if refvar[name][1][4]=='0': dss_group = dss_group0 if refvar[name][1][4]=='1': dss_group = dss_group1 if refvar[name][1][4]=='2': dss_group = dss_group2 ref1 = compare_dss_utils.get_ref(dss_group, refvar[name][1]) p = ref1.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref(ref1) except: print "**** Can't find matched time series for "+ name +" ****" conti = 1 if conti == 0: cpart = p.getPart(p.C_PART).encode('ascii') if name in output_values: ref0_davg = None; ref0_dmax = None; ref0_dmin = None; ref0_mavg = None ref1_davg = None; ref1_dmax = None; ref1_dmin = None; ref1_mavg = None ref2_davg = None; ref2_dmax = None; ref2_dmin = None; ref2_mavg = None if intv < 1000: dataIndex['spec_davg_'+cpart]=dataIndex['spec_davg_'+cpart]+1 dataIndex['spec_dmax_'+cpart]=dataIndex['spec_dmax_'+cpart]+1 dataIndex['spec_dmin_'+cpart]=dataIndex['spec_dmin_'+cpart]+1 if dataIndex['spec_davg_'+cpart]>1: fs_davg[cpart].write(",") if dataIndex['spec_dmax_'+cpart]>1: fs_dmax[cpart].write(",") if dataIndex['spec_dmin_'+cpart]>1: fs_dmin[cpart].write(",") if compare_mode=='1' or compare_mode=='4' or compare_mode=='5': ref0_godin = vtimeseries.godin(ref0) ref0_davg = vtimeseries.per_avg(ref0_godin,'1day') ref0_dmax = vtimeseries.per_max(ref0,'1day') ref0_dmin = vtimeseries.per_min(ref0,'1day') if compare_mode=='2' or compare_mode=='3' or compare_mode=='4' or compare_mode=='5': ref1_godin = vtimeseries.godin(ref1) ref1_davg = vtimeseries.per_avg(ref1_godin,'1day') ref1_dmax = vtimeseries.per_max(ref1,'1day') ref1_dmin = vtimeseries.per_min(ref1,'1day') if compare_mode=='3' or compare_mode=='5': ref2_godin = vtimeseries.godin(ref2) ref2_davg = vtimeseries.per_avg(ref2_godin,'1day') ref2_dmax = vtimeseries.per_max(ref2,'1day') ref2_dmin = vtimeseries.per_min(ref2,'1day') write_plot_data(fs_davg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_davg,ref1_davg,ref2_davg), dataIndex['spec_davg_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Daily Average') write_plot_data(fs_dmax[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_dmax,ref1_dmax,ref2_dmax), dataIndex['spec_dmax_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Daily Maximum') write_plot_data(fs_dmin[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_dmin,ref1_dmin,ref2_dmin), dataIndex['spec_dmin_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Daily Minimum') elif globals['DEFAULT_TIME_INTERVAL']=='1DAY': ref0_davg = ref0; ref0_dmax = ref0; ref0_dmin = ref0 ref1_davg = ref1; ref1_dmax = ref1; ref1_dmin = ref1 ref2_davg = ref2; ref2_dmax = ref2; ref2_dmin = ref2 elif globals['DEFAULT_TIME_INTERVAL']=='1MON': ref2_mavg = ref2; ref2_mmax = ref2; ref2_mmin = ref2 if globals['PLOT_ORIGINAL_TIME_INTERVAL']=='ON': dataIndex['spec_orig_'+cpart]=dataIndex['spec_orig_'+cpart]+1 if dataIndex['spec_orig_'+cpart]>1: fs_orig[cpart].write(",") write_plot_data(fs_orig[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0,ref1,ref2), dataIndex['spec_orig_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Original Time Interval') if intv < 40000: dataIndex['spec_mavg_'+cpart]=dataIndex['spec_mavg_'+cpart]+1 if dataIndex['spec_mavg_'+cpart]>1: fs_mavg[cpart].write(",") if compare_mode=='1' or compare_mode=='4' or compare_mode=='5': ref0_mavg = vtimeseries.per_avg(ref0,'1month') if compare_mode=='2' or compare_mode=='3' or compare_mode=='4' or compare_mode=='5': ref1_mavg = vtimeseries.per_avg(ref1,'1month') if compare_mode=='3' or compare_mode=='5': ref2_mavg = vtimeseries.per_avg(ref2,'1month') write_plot_data(fs_mavg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_mavg,ref1_mavg,ref2_mavg), dataIndex['spec_mavg_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Monthly Average') if (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='ON' and name in output_values): diff_arr = [] if compare_mode=='3' or compare_mode=='4' or compare_mode=='5': for i in range(len(tws)): if compare_mode=='3': rmse_val = vdiff.rmse(ref1, ref2, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse(ref1, ref2, tws[i], time_interval) diff_arr.append([perc_rmse_val,rmse_val ]) if compare_mode=='4': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse(ref0, ref1, tws[i], time_interval) diff_arr.append([perc_rmse_val,rmse_val ]) if compare_mode=='5': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse(ref0, ref1, tws[i], time_interval) rmse_val2 = vdiff.rmse(ref0, ref2, tws[i], time_interval) perc_rmse_val2 = vdiff.perc_rmse(ref0, ref2, tws[i], time_interval) diff_arr.append([perc_rmse_val,rmse_val,perc_rmse_val2,rmse_val2 ]) for w in wy_types: if compare_mode=='3': diff_arr.append([vdiff.rmse_discrete_tws(ref1,ref2,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref1,ref2,wyt_arr[w],1)]) if compare_mode=='4': diff_arr.append([vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],1)]) if compare_mode=='5': diff_arr.append([vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],1), vdiff.rmse_discrete_tws(ref0,ref2,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref0,ref2,wyt_arr[w],1)]) latlng = compare_dss_utils.get_latlng(tbl_latlng,p.getPart(p.B_PART)) if globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='OFF': logging.debug('Working on index: %d/%d '%(dIndex+1,len(refname))) else: logging.debug('Working on index: %d/%d '%(dIndex+1,len(output_values))) dIndex = dIndex + 1 if latlng==None: latlng=['nan','nan','nan','nan'] print "can't find the Lat/Lng for ",p.getPart(p.B_PART) if dIndex>1: fl.write(",") if name in output_values: write_list_data(fl,name, str_refvar, p.getPart(p.B_PART), cpart, 1, diff_arr,latlng) else: write_list_data(fl,name, str_refvar, p.getPart(p.B_PART), cpart, 0, diff_arr,latlng) else: logging.debug("*** Please verify the path for "+name+" ***")
def write_summary_table(fh, dss_group1,dss_group2,globals,scalars,tw_values, var_values, output_values): compare_mode = globals['COMPARE_MODE'] if compare_mode=='1': print >> fh, "<h1><center>Observation Time Series Report<br>Data Set: %s </center></h1>"%(scalars['NAME0']) if compare_mode=='2': print >> fh, "<h1><center>Model Output Time Series Report<br>Study Name: %s </center></h1>"%(scalars['NAME1']) if compare_mode=='3': print >> fh, "<h1><center>DSM2 Output Comparison Report<br> %s <n>vs</n> %s</center></h1>"%(scalars['NAME1'], scalars['NAME2']) if compare_mode=='4': print >> fh, "<h1><center>Calibration Report<br> %s <n>vs</n> %s</center></h1>"%(scalars['NAME1'], scalars['NAME0']) if compare_mode=='5': print >> fh, "<h1><center>Calibration Report<br>[ %s <n>vs</n> %s ] <n> & </n> [ %s <n>vs</n> %s ]</center></h1>"%(scalars['NAME1'],scalars['NAME0'], scalars['NAME2'],scalars['NAME0']) print >> fh, '<div id="note">Note: %s</div>'%(scalars['NOTE'].replace('"','')) print >> fh, '<div id="assumptions">Assumptions: %s</div>'%(scalars['ASSUMPTIONS'].replace('"','')) print >> fh, """<div id="control-panel"> <form> Data Conversion for Plot: <select name="datatype" onChange="change_period()" id="data-conversion"> <option value="davg"> Daily Average</option> <option value="dmax"> Daily Maximum</option> <option value="dmin"> Daily Minimim</option> <option value="mavg"> Monthly Average</option> """ if globals['PLOT_ORIGINAL_TIME_INTERVAL']=='ON': print >> fh, '<option value="orig">Original Data</option>' print >> fh, """ </select><br> Use Defined Time Window: <select name="tw" id="time-window-select"> """ for i in range(len(tw_values)): if i==0: selected = 'selected="selected"' else: selected = "" print >> fh, '<option value="%s" %s>%s</option>'%(compare_dss_utils.timewindow_option_value(vtimeseries.timewindow(tw_values[i][1].replace('"',''))), selected,tw_values[i][0].replace('"','')) print >> fh, """ </select> <div> Customize Time Window: """ twstr = compare_dss_utils.timewindow_option_value(vtimeseries.timewindow(tw_values[0][1].replace('"',''))).split("-") a1=twstr[0].split(",") a2=twstr[1].split(",") print >> fh, """Start Date: <input name="SDate" id="SDate" value="%s" onclick="displayDatePicker('SDate');" size=12>"""%(a1[1]+"/"+a1[2]+"/"+a1[0]) print >> fh, """End Date: <input name="EDate" id="EDate" value="%s" onclick="displayDatePicker('EDate');" size=12>"""%(a2[1]+"/"+a2[2]+"/"+a2[0]) print >> fh, """ <input type=button id="calendar" value="Re-draw"> </div> """ if (compare_mode!='1' and compare_mode!='2'): print >> fh, '<div>' print >> fh, 'Show differences on plot:<input type="checkbox" name="diff_plot" value="1"/>   (Difference = %s - %s)'%(scalars['NAME2'],scalars['NAME1']) print >> fh, '</div>' print >> fh, """ <div> Show water year types on plot: <input type="checkbox" name="wyt" value="1"/>  (<span style="font-size:100%"><span style="background-color:#D9F7CC">    Wet   </span><span style="background-color:#EBF7E6">   Above Normal  </span><span style="background-color:#F7F8F7">  Below Normal  </span><span style="background-color:#FBEEF3">    Dry    </span><span style="background-color:#FFDFEC">   Critical   </span></span>) </div> """ if (compare_mode!='1' and compare_mode!='2'): print >> fh, """ <div> Threshold value to highlight percentage differences <input type="text" id="threshold" value="50"/> </div> <div> Table Statistics: <select name="stat" id="stat" onChange=""> <option value=0>Percentage RMS Diff</option><option value=1>RMS Diff</option> </select> </div> """ print >> fh, """ <div id="warning" style="color:red;font-weight:bold"></div> <input type="hidden" name="ta" id="ta" value=""> </form> </div>""" time_windows = map(lambda val: val[1].replace('"',''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) print >> fh, '<div class="tabber">' if globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='ON': part_c = compare_dss_utils.get_cpart_from_spec(var_values, output_values) else: part_c = compare_dss_utils.get_cpart_list(dss_group1) for type_item in part_c: print >> fh, '<div class="tabbertab" id="%s"><h2>%s</h2><p id="%s_p">'%(type_item,type_item,type_item) print >> fh, '</p></div>' print >> fh, '</div>' print >> fh, '<br><center><font style="font-family:arial,verdana,sans-serif;">Copyright © 2010 State of California, Department of Water Resources<br>Last Modified: 1/25/2011</font></center>' return tws
infile = None outfile = "out.dss" select = "///////" window = None transform = None interval = None for o, a in opts: if o in ("-h", "--help"): usage() return if o in ("-o", "--out"): outfile = a if o in ("-s", "--selection"): select = a if o in ("-w", "--window"): window = timewindow(a) if o in ("-t", "--transform"): transform = a if o in ("-i", "--interval"): interval = timeinterval(a) if not infile: raise "No input file specified" transfer(infile,outfile,select, \ window,transform,interval) # # if __name__ == '__main__': main()
def do_processing(globals, scalars, var_values, output_values, tw_values): compare_mode = globals['COMPARE_MODE'] time_interval = globals['DEFAULT_TIME_INTERVAL'] # open files 1 and file 2 and loop over to plot from java.util import Date dss_group0, dss_group1, dss_group2, refvar, refname = compare_dss_utils.get_group_ref( globals, scalars, var_values) # obtain the new output list modified for *_ or _* cases output_values = compare_dss_utils.get_ref_list(compare_mode, dss_group0, dss_group1, output_values) output_dir = scalars['OUTDIR'].replace('"', '') wyt_arr = write_wyt(output_dir, tw_values) #write the water type JavaScript file #out_name = column(output_values,0) #out_type = column(output_values,1) if compare_mode == '1': type_arr = compare_dss_utils.get_cpart_list(dss_group0) else: type_arr = compare_dss_utils.get_cpart_list(dss_group1) compare_dss_utils.chk_cpart_space(type_arr) if output_dir[-1] != '/': output_dir = output_dir + "/" time_windows = map(lambda val: val[1].replace('"', ''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) dIndex = 0 dataIndex = {} fs_davg = {} fs_dmax = {} fs_dmin = {} fs_mavg = {} fs_orig = {} tbl_latlng = compare_dss_utils.lines2table(output_dir + "js/latlng.txt") fl = open(output_dir + "data/data_list.js", 'w') for i in type_arr: initial_js = output_dir + "data/DSS_compare_spec_davg_" + i + ".js" initial_pretab = i fs_davg[i] = open( output_dir + "data/DSS_compare_spec_davg_" + i + ".js", 'w') fs_dmax[i] = open( output_dir + "data/DSS_compare_spec_dmax_" + i + ".js", 'w') fs_dmin[i] = open( output_dir + "data/DSS_compare_spec_dmin_" + i + ".js", 'w') fs_mavg[i] = open( output_dir + "data/DSS_compare_spec_mavg_" + i + ".js", 'w') print >> fs_davg[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) print >> fs_dmax[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) print >> fs_dmin[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) print >> fs_mavg[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) dataIndex['spec_davg_' + i] = 0 dataIndex['spec_dmax_' + i] = 0 dataIndex['spec_dmin_' + i] = 0 dataIndex['spec_mavg_' + i] = 0 js_data.write_begin_data_array(fs_davg[i]) js_data.write_begin_data_array(fs_dmax[i]) js_data.write_begin_data_array(fs_dmin[i]) js_data.write_begin_data_array(fs_mavg[i]) if globals['PLOT_ORIGINAL_TIME_INTERVAL'] == 'ON': fs_orig[i] = open( output_dir + "data/DSS_compare_spec_orig_" + i + ".js", 'w') print >> fs_orig[ i], """/* Comparison Output File Generated on : %s */""" % ( str(Date())) dataIndex['spec_orig_' + i] = 0 js_data.write_begin_data_array(fs_orig[i]) js_data_list.write_begin_data_array(fl) wy_types = ['W', 'AN', 'BN', 'D', 'C'] if globals['DONOT_SORT_STATION_NAME'] == 'OFF': output_values = compare_dss_utils.sort_alphabetically(output_values) refname = compare_dss_utils.sort_alphabetically(refname) else: refname = compare_dss_utils.get_not_sort_refname( refname, output_values) for name in refname: ref0 = None ref1 = None ref2 = None try: series_name = compare_dss_utils.get_series_name( compare_mode, scalars, refvar[name]) except Exception, e: logging.error("Error building name for :%s and error is %s" % (name, e)) continue if (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'ON' and name in output_values): conti = 0 str_refvar = compare_dss_utils.get_str_refvar( refvar[name], compare_mode) if compare_mode == '1' or compare_mode == '4' or compare_mode == '5': try: if refvar[name][0][4] == '0': dss_group = dss_group0 if refvar[name][0][4] == '1': dss_group = dss_group1 if refvar[name][0][4] == '2': dss_group = dss_group2 ref0 = compare_dss_utils.get_ref(dss_group, refvar[name][0]) p = ref0.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref( ref0) except: print "**** Can't find matched time series for " + name + " ****" conti = 1 if compare_mode == '3' or compare_mode == '5': try: if refvar[name][2][4] == '0': dss_group = dss_group0 if refvar[name][2][4] == '1': dss_group = dss_group1 if refvar[name][2][4] == '2': dss_group = dss_group2 ref2 = compare_dss_utils.get_ref(dss_group, refvar[name][2]) p = ref2.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref( ref2) except: print "**** Can't find matched time series for " + name + " ****" conti = 1 if compare_mode == '2' or compare_mode == '3' or compare_mode == '4' or compare_mode == '5': try: if refvar[name][1][4] == '0': dss_group = dss_group0 if refvar[name][1][4] == '1': dss_group = dss_group1 if refvar[name][1][4] == '2': dss_group = dss_group2 ref1 = compare_dss_utils.get_ref(dss_group, refvar[name][1]) p = ref1.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref( ref1) except: print "**** Can't find matched time series for " + name + " ****" conti = 1 if conti == 0: cpart = p.getPart(p.C_PART).encode('ascii') if name in output_values: ref0_davg = None ref0_dmax = None ref0_dmin = None ref0_mavg = None ref1_davg = None ref1_dmax = None ref1_dmin = None ref1_mavg = None ref2_davg = None ref2_dmax = None ref2_dmin = None ref2_mavg = None if intv < 1000: dataIndex['spec_davg_' + cpart] = dataIndex['spec_davg_' + cpart] + 1 dataIndex['spec_dmax_' + cpart] = dataIndex['spec_dmax_' + cpart] + 1 dataIndex['spec_dmin_' + cpart] = dataIndex['spec_dmin_' + cpart] + 1 if dataIndex['spec_davg_' + cpart] > 1: fs_davg[cpart].write(",") if dataIndex['spec_dmax_' + cpart] > 1: fs_dmax[cpart].write(",") if dataIndex['spec_dmin_' + cpart] > 1: fs_dmin[cpart].write(",") if compare_mode == '1' or compare_mode == '4' or compare_mode == '5': ref0_godin = vtimeseries.godin(ref0) ref0_davg = vtimeseries.per_avg(ref0_godin, '1day') ref0_dmax = vtimeseries.per_max(ref0, '1day') ref0_dmin = vtimeseries.per_min(ref0, '1day') if compare_mode == '2' or compare_mode == '3' or compare_mode == '4' or compare_mode == '5': ref1_godin = vtimeseries.godin(ref1) ref1_davg = vtimeseries.per_avg(ref1_godin, '1day') ref1_dmax = vtimeseries.per_max(ref1, '1day') ref1_dmin = vtimeseries.per_min(ref1, '1day') if compare_mode == '3' or compare_mode == '5': ref2_godin = vtimeseries.godin(ref2) ref2_davg = vtimeseries.per_avg(ref2_godin, '1day') ref2_dmax = vtimeseries.per_max(ref2, '1day') ref2_dmin = vtimeseries.per_min(ref2, '1day') write_plot_data( fs_davg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_davg, ref1_davg, ref2_davg), dataIndex['spec_davg_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Daily Average') write_plot_data( fs_dmax[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_dmax, ref1_dmax, ref2_dmax), dataIndex['spec_dmax_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Daily Maximum') write_plot_data( fs_dmin[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_dmin, ref1_dmin, ref2_dmin), dataIndex['spec_dmin_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Daily Minimum') elif globals['DEFAULT_TIME_INTERVAL'] == '1DAY': ref0_davg = ref0 ref0_dmax = ref0 ref0_dmin = ref0 ref1_davg = ref1 ref1_dmax = ref1 ref1_dmin = ref1 ref2_davg = ref2 ref2_dmax = ref2 ref2_dmin = ref2 elif globals['DEFAULT_TIME_INTERVAL'] == '1MON': ref2_mavg = ref2 ref2_mmax = ref2 ref2_mmin = ref2 if globals['PLOT_ORIGINAL_TIME_INTERVAL'] == 'ON': dataIndex['spec_orig_' + cpart] = dataIndex['spec_orig_' + cpart] + 1 if dataIndex['spec_orig_' + cpart] > 1: fs_orig[cpart].write(",") write_plot_data( fs_orig[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0, ref1, ref2), dataIndex['spec_orig_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Original Time Interval') if intv < 40000: dataIndex['spec_mavg_' + cpart] = dataIndex['spec_mavg_' + cpart] + 1 if dataIndex['spec_mavg_' + cpart] > 1: fs_mavg[cpart].write(",") if compare_mode == '1' or compare_mode == '4' or compare_mode == '5': ref0_mavg = vtimeseries.per_avg(ref0, '1month') if compare_mode == '2' or compare_mode == '3' or compare_mode == '4' or compare_mode == '5': ref1_mavg = vtimeseries.per_avg(ref1, '1month') if compare_mode == '3' or compare_mode == '5': ref2_mavg = vtimeseries.per_avg(ref2, '1month') write_plot_data( fs_mavg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_mavg, ref1_mavg, ref2_mavg), dataIndex['spec_mavg_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Monthly Average') if (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'ON' and name in output_values): diff_arr = [] if compare_mode == '3' or compare_mode == '4' or compare_mode == '5': for i in range(len(tws)): if compare_mode == '3': rmse_val = vdiff.rmse(ref1, ref2, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse( ref1, ref2, tws[i], time_interval) diff_arr.append([perc_rmse_val, rmse_val]) if compare_mode == '4': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse( ref0, ref1, tws[i], time_interval) diff_arr.append([perc_rmse_val, rmse_val]) if compare_mode == '5': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse( ref0, ref1, tws[i], time_interval) rmse_val2 = vdiff.rmse(ref0, ref2, tws[i], time_interval) perc_rmse_val2 = vdiff.perc_rmse( ref0, ref2, tws[i], time_interval) diff_arr.append([ perc_rmse_val, rmse_val, perc_rmse_val2, rmse_val2 ]) for w in wy_types: if compare_mode == '3': diff_arr.append([ vdiff.rmse_discrete_tws( ref1, ref2, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref1, ref2, wyt_arr[w], 1) ]) if compare_mode == '4': diff_arr.append([ vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 1) ]) if compare_mode == '5': diff_arr.append([ vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 1), vdiff.rmse_discrete_tws( ref0, ref2, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref0, ref2, wyt_arr[w], 1) ]) latlng = compare_dss_utils.get_latlng( tbl_latlng, p.getPart(p.B_PART)) if globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'OFF': logging.debug('Working on index: %d/%d ' % (dIndex + 1, len(refname))) else: logging.debug('Working on index: %d/%d ' % (dIndex + 1, len(output_values))) dIndex = dIndex + 1 if latlng == None: latlng = ['nan', 'nan', 'nan', 'nan'] print "can't find the Lat/Lng for ", p.getPart( p.B_PART) if dIndex > 1: fl.write(",") if name in output_values: write_list_data(fl, name, str_refvar, p.getPart(p.B_PART), cpart, 1, diff_arr, latlng) else: write_list_data(fl, name, str_refvar, p.getPart(p.B_PART), cpart, 0, diff_arr, latlng) else: logging.debug("*** Please verify the path for " + name + " ***")
def planning_ec_mtz(): # MTZ = RSAC054 BC for the qual DEBUG = 0 OUTPUT=config.getAttr('QUALBOUNDARYFILE') calsimfile = config.getAttr('CALSIMFILE') vamp_corrected_dss = config.getAttr('CALSIM_VAMP') CALSIM=opendss(calsimfile) PLANNINGTIDE=opendss(config.getAttr('STAGE_SOURCE_FILE')) STEP=string.lower(config.getAttr('CALSIMSTEP')) SJR_PROCESS=config.getAttr("SJR_PROCESS") outputpath="/FILL+CHAN/RSAC054/EC//15MIN/"+config.getAttr("DSM2MODIFIER")+"/" if not(OUTPUT and os.path.exists(OUTPUT)): raise "Envvar QUALBOUNDARYFILE must exist as destination for EC" startyr=int(config.getAttr('START_DATE')[5:]) endyr=int(config.getAttr('END_DATE')[5:]) if (startyr < 1974 and endyr > 1991): blocks= [ "01NOV1921 0000 - 01OCT1940 0000", "01OCT1940 0000 - 01OCT1960 0000", "01OCT1960 0000 - 01OCT1974 0000", "01OCT1974 0000 - 01OCT1991 0000", "01OCT1991 0000 - 01OCT2003 0000" ] else: blocks = [ "01OCT1974 0000 - 01OCT1991 0000" ] # for memory reasons (year 2001). g0=5000. # initial value of g (antecedent outflow) for the beginning # of the first year. This is pretty arbitrary and makes little difference if DEBUG: g0_no_vamp = 5000. for twstr in blocks: TWIND=timewindow(twstr) # Actual period to be estimated print "Calculating boundary salinity for the period "+TWIND.toString() TWINDBUF=grow_window(TWIND,"1MON","1MON") # Conservative buffered period for retrieval # so that after prelimiary operations (e.g. time average) # time series will still span at least TWIND fpart=calsim_study_fpart(modify=0) ndo=DataReference.create(findpath(CALSIM,"/CALSIM/NDO/FLOW-NDO//"+STEP+"/" +fpart+"/")[0],TWIND).getData() ndo15=conserve.conserveSpline(ndo,"15MIN") ndo15_no_vamp = 0 if DEBUG: ndo15_no_vamp = ndo15 # calc vamp caused ndo change if (SJR_PROCESS.upper()=="SINGLE_STEP") or (SJR_PROCESS.upper()=="MULTI_STEP"): fpart_modified=calsim_study_fpart(modify=1) delta_ndo = vamp_ndo.calc_vamp_delta_ndo(calsimfile,vamp_corrected_dss,fpart,fpart_modified,SJR_PROCESS) ndo15 = ndo15 + interpolate(delta_ndo, "15MIN") astro_stage_version = config.getAttr("ASTRO_STAGE_VERSION") mtzastro=DataReference.create(findpath(PLANNINGTIDE,"/FILL\+CHAN/RSAC054/STAGE//15MIN/"+astro_stage_version + "/")[0],TWINDBUF).getData() astrorms=godin((mtzastro*mtzastro)**0.5) # RMS energy of tide (used to predict filling and draining) dastrorms=( (astrorms-(astrorms>>1))*96. ).createSlice(TWIND) fifteenflo2=ndo15 - 40000*(dastrorms) # call to ec estimator. all parameters are included. g0 is an [mtzecest, g1]=ec_boundary.ECEst(mtzastro,fifteenflo2,beta=600,npow1=0.75,npow2=1,g0=g0,zrms=astrorms) if DEBUG: fifteenflo2_no_vamp = ndo15_no_vamp - 40000*(dastrorms) [mtzecest_no_vamp, g1_no_vamp]=ec_boundary.ECEst(mtzastro,fifteenflo2_no_vamp,beta=600,npow1=0.75,npow2=1,g0=g0_no_vamp,zrms=astrorms) g0_no_vamp = g1_no_vamp writedss("out_ec_check","/CALC/ndo_no_vamp/ndo////", ndo15_no_vamp) writedss("out_ec_check","/CALC/ndo_with_vamp/ndo////", ndo15) writedss("out_ec_check","/CALC/ndo_no_vamp/ec////", mtzecest_no_vamp) writedss("out_ec_check","/CALC/ndo_with_vamp/ec////", mtzecest) writedss(OUTPUT,outputpath,mtzecest) g0=g1 return 0
infile = None outfile="out.dss" select="///////" window=None transform=None interval=None for o, a in opts: if o in ("-h", "--help"): usage() return if o in ("-o", "--out"): outfile=a if o in ("-s", "--selection"): select=a if o in ("-w", "--window"): window=timewindow(a) if o in ("-t", "--transform"): transform=a if o in ("-i", "--interval"): interval = timeinterval(a) if not infile: raise "No input file specified" transfer(infile,outfile,select, \ window,transform,interval) # # if __name__=='__main__': main() sys.exit()
def planning_ec_mtz(): # MTZ = RSAC054 BC for the qual DEBUG = 0 OUTPUT = config.getAttr('QUALBOUNDARYFILE') calsimfile = config.getAttr('CALSIMFILE') CALSIM = opendss(calsimfile) PLANNINGTIDE = opendss(config.getAttr('STAGE_SOURCE_FILE')) outputpath = "/FILL+CHAN/RSAC054/EC//15MIN/" + config.getAttr( "DSM2MODIFIER") + "/" if not (OUTPUT and os.path.exists(OUTPUT)): raise "Envvar QUALBOUNDARYFILE must exist as destination for EC" startyr = int(config.getAttr('START_DATE')[5:]) endyr = int(config.getAttr('END_DATE')[5:]) if (startyr < 1974 and endyr > 1991): blocks = [ "01NOV1921 0000 - 01OCT1940 0000", "01OCT1940 0000 - 01OCT1960 0000", "01OCT1960 0000 - 01OCT1974 0000", "01OCT1974 0000 - 01OCT1991 0000", "01OCT1991 0000 - 01OCT2003 0000" ] # for memory reasons (year 2001). else: blocks = ["01OCT1974 0000 - 01OCT1991 0000"] g0 = 5000. # initial value of g (antecedent outflow) for the beginning # of the first year. This is pretty arbitrary and makes little difference for twstr in blocks: TWIND = timewindow(twstr) # Actual period to be estimated print "Calculating boundary salinity for the period " + TWIND.toString( ) TWINDBUF = grow_window( TWIND, "1MON", "1MON") # Conservative buffered period for retrieval # so that after prelimiary operations (e.g. time average) # time series will still span at least TWIND fpart = calsim_study_fpart(modify=0) ndo15 = DataReference.create( findpath(CALSIM, "/CALSIM/NDO/FLOW-NDO//15MIN/" + fpart + "/")[0], TWIND).getData() astro_stage_version = config.getAttr("ASTRO_STAGE_VERSION") mtzastro = DataReference.create( findpath( PLANNINGTIDE, "/FILL\+CHAN/RSAC054/STAGE//15MIN/" + astro_stage_version + "/")[0], TWINDBUF).getData() if 'NAVD' in astro_stage_version: mtzastro = mtzastro - 2.68 writedss( CALSIM, "/FILL\+CHAN/RSAC054/STAGE//15MIN/" + astro_stage_version.replace('NAVD', 'NGVD') + "/", mtzastro) astrorms = godin( (mtzastro * mtzastro)** 0.5) # RMS energy of tide (used to predict filling and draining) dastrorms = ((astrorms - (astrorms >> 1)) * 96.).createSlice(TWIND) fifteenflo2 = ndo15 - 53411.1 * (dastrorms) # call to ec estimator. all parameters are included. so, sb = 37196, 2328.1 c = [ -6.00E-05, 7.30E-05, -1.00E-05, -3.00E-05, 1.70E-06, -1.00E-04, 4.50E-05, -1.00E-04 ] [mtzecest, g1] = ec_boundary.ECEst(mtzastro, fifteenflo2, so, sb, beta=420.5205, npow1=0.7750588, npow2=1, g0=g0, zrms=astrorms, c=c) writedss(OUTPUT, outputpath, mtzecest) g0 = g1 return 0
def planning_ec_mtz(): # MTZ = RSAC054 BC for the qual DEBUG = 0 OUTPUT=config.getAttr('QUALBOUNDARYFILE') calsimfile = config.getAttr('CALSIMFILE') vamp_corrected_dss = config.getAttr('CALSIM_VAMP') CALSIM=opendss(calsimfile) PLANNINGTIDE=opendss(config.getAttr('STAGE_SOURCE_FILE')) STEP=string.lower(config.getAttr('CALSIMSTEP')) SJR_PROCESS=config.getAttr("SJR_PROCESS") outputpath="/FILL+CHAN/RSAC054/EC//15MIN/"+config.getAttr("DSM2MODIFIER")+"/" if not(OUTPUT and os.path.exists(OUTPUT)): raise "Envvar QUALBOUNDARYFILE must exist as destination for EC" startyr=int(config.getAttr('START_DATE')[5:]) endyr=int(config.getAttr('END_DATE')[5:]) if (startyr < 1974 and endyr > 1991): # blocks= [ "01NOV1921 0000 - 01OCT1940 0000", blocks= [ "01FEB1921 0000 - 01OCT1940 0000", "01OCT1940 0000 - 01OCT1960 0000", "01OCT1960 0000 - 01OCT1974 0000", "01OCT1974 0000 - 01OCT1991 0000", "01OCT1991 0000 - 01OCT2003 0000" ] else: blocks = [ "01OCT1974 0000 - 01OCT1991 0000" ] # for memory reasons (year 2001). g0=5000. # initial value of g (antecedent outflow) for the beginning # of the first year. This is pretty arbitrary and makes little difference if DEBUG: g0_no_vamp = 5000. for twstr in blocks: TWIND=timewindow(twstr) # Actual period to be estimated print "Calculating boundary salinity for the period "+TWIND.toString() TWINDBUF=grow_window(TWIND,"1MON","1MON") # Conservative buffered period for retrieval # so that after prelimiary operations (e.g. time average) # time series will still span at least TWIND fpart=calsim_study_fpart(modify=0) ndo=DataReference.create(findpath(CALSIM,"/CALSIM/NDO/FLOW-NDO//"+STEP+"/" +fpart+"/")[0],TWIND).getData() ndo15=conserve.conserveSpline(ndo,"15MIN") ndo15_no_vamp = 0 if DEBUG: ndo15_no_vamp = ndo15 # removed since currently there is no difference between vamp and monthly average. # calc vamp caused ndo change # if (SJR_PROCESS.upper()=="SINGLE_STEP") or (SJR_PROCESS.upper()=="MULTI_STEP"): # fpart_modified=calsim_study_fpart(modify=1) # delta_ndo = vamp_ndo.calc_vamp_delta_ndo(calsimfile,vamp_corrected_dss,fpart,fpart_modified,SJR_PROCESS) # ndo15 = ndo15 + interpolate(delta_ndo, "15MIN") astro_stage_version = config.getAttr("ASTRO_STAGE_VERSION") mtzastro=DataReference.create(findpath(PLANNINGTIDE,"/FILL\+CHAN/RSAC054/STAGE//15MIN/"+astro_stage_version + "/")[0],TWINDBUF).getData() astrorms=godin((mtzastro*mtzastro)**0.5) # RMS energy of tide (used to predict filling and draining) dastrorms=( (astrorms-(astrorms>>1))*96. ).createSlice(TWIND) fifteenflo2=ndo15 - 40000*(dastrorms) # call to ec estimator. all parameters are included. g0 is an [mtzecest, g1]=ec_boundary.ECEst(mtzastro,fifteenflo2,beta=600,npow1=0.75,npow2=1,g0=g0,zrms=astrorms) if DEBUG: fifteenflo2_no_vamp = ndo15_no_vamp - 40000*(dastrorms) [mtzecest_no_vamp, g1_no_vamp]=ec_boundary.ECEst(mtzastro,fifteenflo2_no_vamp,beta=600,npow1=0.75,npow2=1,g0=g0_no_vamp,zrms=astrorms) g0_no_vamp = g1_no_vamp writedss("out_ec_check","/CALC/ndo_no_vamp/ndo////", ndo15_no_vamp) writedss("out_ec_check","/CALC/ndo_with_vamp/ndo////", ndo15) writedss("out_ec_check","/CALC/ndo_no_vamp/ec////", mtzecest_no_vamp) writedss("out_ec_check","/CALC/ndo_with_vamp/ec////", mtzecest) writedss(OUTPUT,outputpath,mtzecest) g0=g1 return 0
def write_summary_table(fh, dss_group1, dss_group2, globals, scalars, tw_values, var_values, output_values): compare_mode = globals['COMPARE_MODE'] if compare_mode == '1': print >> fh, "<h1><center>Observation Time Series Report<br>Data Set: %s </center></h1>" % ( scalars['NAME0']) if compare_mode == '2': print >> fh, "<h1><center>Model Output Time Series Report<br>Study Name: %s </center></h1>" % ( scalars['NAME1']) if compare_mode == '3': print >> fh, "<h1><center>DSM2 Output Comparison Report<br> %s <n>vs</n> %s</center></h1>" % ( scalars['NAME1'], scalars['NAME2']) if compare_mode == '4': print >> fh, "<h1><center>Calibration Report<br> %s <n>vs</n> %s</center></h1>" % ( scalars['NAME1'], scalars['NAME0']) if compare_mode == '5': print >> fh, "<h1><center>Calibration Report<br>[ %s <n>vs</n> %s ] <n> & </n> [ %s <n>vs</n> %s ]</center></h1>" % ( scalars['NAME1'], scalars['NAME0'], scalars['NAME2'], scalars['NAME0']) print >> fh, '<div id="note">Note: %s</div>' % (scalars['NOTE'].replace( '"', '')) print >> fh, '<div id="assumptions">Assumptions: %s</div>' % ( scalars['ASSUMPTIONS'].replace('"', '')) print >> fh, """<div id="control-panel"> <form> Data Conversion for Plot: <select name="datatype" onChange="change_period()" id="data-conversion"> <option value="davg"> Daily Average</option> <option value="dmax"> Daily Maximum</option> <option value="dmin"> Daily Minimim</option> <option value="mavg"> Monthly Average</option> """ if globals['PLOT_ORIGINAL_TIME_INTERVAL'] == 'ON': print >> fh, '<option value="orig">Original Data</option>' print >> fh, """ </select><br> Use Defined Time Window: <select name="tw" id="time-window-select"> """ for i in range(len(tw_values)): if i == 0: selected = 'selected="selected"' else: selected = "" print >> fh, '<option value="%s" %s>%s</option>' % ( compare_dss_utils.timewindow_option_value( vtimeseries.timewindow(tw_values[i][1].replace( '"', ''))), selected, tw_values[i][0].replace('"', '')) print >> fh, """ </select> <div> Customize Time Window: """ twstr = compare_dss_utils.timewindow_option_value( vtimeseries.timewindow(tw_values[0][1].replace('"', ''))).split("-") a1 = twstr[0].split(",") a2 = twstr[1].split(",") print >> fh, """Start Date: <input name="SDate" id="SDate" value="%s" onclick="displayDatePicker('SDate');" size=12>""" % ( a1[1] + "/" + a1[2] + "/" + a1[0]) print >> fh, """End Date: <input name="EDate" id="EDate" value="%s" onclick="displayDatePicker('EDate');" size=12>""" % ( a2[1] + "/" + a2[2] + "/" + a2[0]) print >> fh, """ <input type=button id="calendar" value="Re-draw"> </div> """ if (compare_mode != '1' and compare_mode != '2'): print >> fh, '<div>' print >> fh, 'Show differences on plot:<input type="checkbox" name="diff_plot" value="1"/>   (Difference = %s - %s)' % ( scalars['NAME2'], scalars['NAME1']) print >> fh, '</div>' print >> fh, """ <div> Show water year types on plot: <input type="checkbox" name="wyt" value="1"/>  (<span style="font-size:100%"><span style="background-color:#D9F7CC">    Wet   </span><span style="background-color:#EBF7E6">   Above Normal  </span><span style="background-color:#F7F8F7">  Below Normal  </span><span style="background-color:#FBEEF3">    Dry    </span><span style="background-color:#FFDFEC">   Critical   </span></span>) </div> """ if (compare_mode != '1' and compare_mode != '2'): print >> fh, """ <div> Threshold value to highlight percentage differences <input type="text" id="threshold" value="50"/> </div> <div> Table Statistics: <select name="stat" id="stat" onChange=""> <option value=0>Percentage RMS Diff</option><option value=1>RMS Diff</option> </select> </div> """ print >> fh, """ <div id="warning" style="color:red;font-weight:bold"></div> <input type="hidden" name="ta" id="ta" value=""> </form> </div>""" time_windows = map(lambda val: val[1].replace('"', ''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) print >> fh, '<div class="tabber">' if globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'ON': part_c = compare_dss_utils.get_cpart_from_spec(var_values, output_values) else: part_c = compare_dss_utils.get_cpart_list(dss_group1) for type_item in part_c: print >> fh, '<div class="tabbertab" id="%s"><h2>%s</h2><p id="%s_p">' % ( type_item, type_item, type_item) print >> fh, '</p></div>' print >> fh, '</div>' print >> fh, '<br><center><font style="font-family:arial,verdana,sans-serif;">Copyright © 2010 State of California, Department of Water Resources<br>Last Modified: 1/25/2011</font></center>' return tws
def ECEst(stage, ndo, beta, npow1, npow2,g0=None, zrms=None, c=getMTZCoef()): """ Estimate 15min EC at the boundary. Inputs: stage astronomical tide estimate. Only 15min data are acceptable ndo ndo estimate -- e.g., from CALSIM """ import interpolate from vista.set import Units if not isinstance(stage,RegularTimeSeries) or \ not isinstance(ndo,RegularTimeSeries): raise "stage and ndo must be RegularTimeSeries" if ndo.getTimeInterval().toString() == "1DAY": ndo=interpolate.spline(ndo,"15MIN",0.5) << 95 elif ndo.getTimeInterval().toString()!= "15MIN": raise "ndo must be a one day or 15 minute series" if not stage.getTimeInterval().toString() == "15MIN": raise "stage must be an hourly or 15 minute series" # if ndo.getTimeInterval().toString() != stage.getTimeInterval().toString(): raise "stage and ndo must have the same window" # #if not len(c) ==9: # raise "Wrong number (%s) of coefficients in the array c" % len(c) if (first_missing(ndo)) >= 0: raise "missing data not allowed in ndo. First missing data at index: %s" % first_missing(ndo) if (first_missing(stage)) >= 0: raise "missing data not allowed in stage. First missing data at index: %s" % first_missing(stage) newstart=ndo.getStartTime() - "21HOURS" newend =ndo.getEndTime() - "3HOURS" if (stage.getStartTime().getTimeInMinutes() - newstart.getTimeInMinutes() > 0): print "Stage record starts %s and NDO starts %s" % (stage.getStartTime().toString(),ndo.getStartTime().toString()) raise "stage record must begin at least 21 hours before ndo" if (newend.getTimeInMinutes() - stage.getEndTime().getTimeInMinutes() > 0): raise "stage record must end no more than 3 hours before end of ndo" ztw=timewindow(newstart.toString()+' - '+newend.toString()) z=stage.createSlice(ztw) g=gCalc(ndo,beta,g0) zarr=z.getYArray() giter=g.getIterator() ec=zeros(g.size(),'d') ec=map(lambda x: -901.0, ec) so=32797 #ocean salinity parameter for gmodel sb=200 #river salinity parameter for gmodel zrmsiter=zrms.getIterator() i=0 while (not giter.atEnd()): gval=giter.getElement().getY() zrmsval=zrmsiter.getElement().getY() ecfrac = c[0] + gval*c[1] + 1.1*gval**npow1*(c[2]*zarr[i+72] + c[3]*zarr[i+60] + c[4]*zarr[i+48] + c[5]*zarr[i+36] + c[6]*+zarr[i+24] + c[7]*zarr[i+12] + c[8]*zarr[i])-0.1 #+ c[9]*zrmsval ec[i]=max(200,exp(ecfrac)*(so-sb) + sb) #ec[i]=max((1+ecfrac)*(so-sb)+sb,200) zrmsiter.advance() giter.advance() i=i+1 # ec needs to be set umhos/cm rts = RegularTimeSeries("/ECest//////",g.getStartTime().toString(),g.getTimeInterval().toString(),ec) rts.getAttributes().setYUnits(Units.UMHOS_CM) return [rts,gval]