def do_processing(scalars, pathname_maps, tw_values): # open files 1 and file 2 and loop over to plot dss_group1 = vutils.opendss(scalars['FILE1']) dss_group2 = vutils.opendss(scalars['FILE2']) time_windows = map(lambda val: val[1].replace('"',''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) if len(tws) > 0: tw=tws[0] else: tw=None output_file=scalars['OUTFILE'] data_output_file = output_file.split(".")[0]+".js" fh=open(data_output_file,'w') print >> fh, """/* Comparison Output File Generated on : %s */"""%(str(Date())) js_data.write_begin_data_array(fh); if dss_group1 == None or dss_group2 == None: sys.exit(2); dataIndex=0 for path_map in pathname_maps: dataIndex=dataIndex+1 logging.debug('Working on index: %d'%dataIndex) if dataIndex>1: fh.write(",") #path_map = pathname_mapping[var_name] if path_map.path2==None or path_map.path2 == "": path_map.path2=path_map.path1 var_name = path_map.var_name calculate_dts=0 if path_map.var_category == 'HEADER': logging.debug('Inserting header') continue; if path_map.report_type == 'Exceedance_Post': calculate_dts=1 ref1 = get_ref(dss_group1, path_map.path1,calculate_dts, pathname_maps, 1) ref2 = get_ref(dss_group2, path_map.path2,calculate_dts, pathname_maps, 2) if (ref1==None or ref2==None): continue series_name = [scalars['NAME1'],scalars['NAME2']] data_units=get_units(ref1,ref2) data_type=get_type(ref1,ref2) if path_map.report_type == 'Average': write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES) elif path_map.report_type == 'Exceedance': write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE) elif path_map.report_type == 'Avg_Excd': write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES) fh.write(",") write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE) elif path_map.report_type == 'Timeseries': write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES) elif path_map.report_type == 'Exceedance_Post': write_plot_data(fh, build_exceedance_array(ref1,ref2,tw), dataIndex, "Exceedance %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE) js_data.write_end_data_array(fh); logging.debug('Writing end of data array') fh.close() # Generate the main html file fh=open(scalars['OUTFILE'],'w') print >> fh, """ <html> <head> <title>Calsim Report: %s vs %s</title> <script type="text/javascript" src="%s"></script> <script type="text/javascript" src="report-common/protovis-d3.3.js"></script> <script type="text/javascript" src="report-common/plots.js"></script> <script type="text/javascript" src="report-common/jquery-1.4.2.min.js"></script> <link rel="stylesheet" type="text/css" media="print" href="report-common/print.css" /> <link rel="stylesheet" type="text/css" media="screen" href="report-common/screen.css" /> </head> """%(scalars['NAME1'],scalars['NAME2'],data_output_file) write_control_panel(fh,scalars, pathname_maps,tw_values) write_water_balance_table(fh,dss_group1, dss_group2, scalars, pathname_maps,tw_values) print >> fh, """<script type="text/javascript"> function clear_and_draw(sdate, edate){ $('.plot').empty(); n=data.length plot_diff = $('input[name=diff_plot]').is(':checked') ? 1 : 0 ; for(i=0; i < n; i++){ var div_id = "fig"+(i+1); if (data[i]==null) continue; if ($("#"+div_id).length==0){ $("body").append('<a href="#'+div_id+'"><div class="plot" id="'+div_id+'"></div></a>'); } if (data[i].plot_type=="timeseries"){ plots.time_series_plot(div_id,data[i],plot_diff,sdate,edate); }else if (data[i].plot_type=="exceedance"){ plots.exceedance_plot(div_id,data[i],null,sdate,edate); } } // update links }; $(document).ready(clear_and_draw(null,null)); $('#system-water-balance-table tr').click(function(){ var_name = $($(this).find('td')[0]).text(); var svg_element = $('div').filter(function(){ return $(this).text().indexOf(var_name)>=0;}) if (svg_element && svg_element.length > 0){ anchor_name = $($(svg_element[0]).parent()).attr("href"); window.location.href=window.location.href.split('#')[0]+anchor_name; } }); </script> <script type="text/javascript"> function extract_date(date_str){ date_fields=date_str.split(","); return new Date(date_fields[0],date_fields[1],date_fields[2]); } $('#time-window-select').change(function(){ var changed_val = $('#time-window-select option:selected').val().split("-"); clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1])); }); $('#threshold').change(function(){ set_diff_threshold($('#threshold').val()); }); $('input[name=diff_plot]').change(function(){ var changed_val = $('#time-window-select option:selected').val().split("-"); clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1])); }); function set_diff_threshold(threshold){ var diff_marker = function(){ if(Math.abs(parseFloat($(this).text())) >= threshold){ $(this).addClass('large-diff'); }else{ $(this).removeClass('large-diff'); }; }; $('tr td:nth-child(5)').each(diff_marker); $('tr td:nth-child(9)').each(diff_marker); $('tr td:nth-child(13)').each(diff_marker); }; set_diff_threshold($('#threshold').val()); </script> </body> </html>""" fh.close() logging.debug('Closed out data file')
def do_processing(globals, scalars, var_values, output_values, tw_values): compare_mode = globals['COMPARE_MODE'] time_interval = globals['DEFAULT_TIME_INTERVAL'] # open files 1 and file 2 and loop over to plot from java.util import Date dss_group0, dss_group1, dss_group2, refvar, refname = compare_dss_utils.get_group_ref(globals, scalars, var_values) # obtain the new output list modified for *_ or _* cases output_values = compare_dss_utils.get_ref_list(compare_mode,dss_group0,dss_group1,output_values) output_dir = scalars['OUTDIR'].replace('"','') wyt_arr = write_wyt(output_dir, tw_values) #write the water type JavaScript file #out_name = column(output_values,0) #out_type = column(output_values,1) if compare_mode=='1': type_arr = compare_dss_utils.get_cpart_list(dss_group0) else: type_arr = compare_dss_utils.get_cpart_list(dss_group1) compare_dss_utils.chk_cpart_space(type_arr) if output_dir[-1]!='/': output_dir=output_dir+"/" time_windows = map(lambda val: val[1].replace('"',''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) dIndex = 0 dataIndex = {} fs_davg = {} fs_dmax = {} fs_dmin = {} fs_mavg = {} fs_orig = {} tbl_latlng = compare_dss_utils.lines2table(output_dir+"js/latlng.txt") fl = open(output_dir+"data/data_list.js",'w') for i in type_arr: initial_js = output_dir+"data/DSS_compare_spec_davg_"+i+".js" initial_pretab = i fs_davg[i] = open(output_dir+"data/DSS_compare_spec_davg_"+i+".js",'w') fs_dmax[i] = open(output_dir+"data/DSS_compare_spec_dmax_"+i+".js",'w') fs_dmin[i] = open(output_dir+"data/DSS_compare_spec_dmin_"+i+".js",'w') fs_mavg[i] = open(output_dir+"data/DSS_compare_spec_mavg_"+i+".js",'w') print >> fs_davg[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) print >> fs_dmax[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) print >> fs_dmin[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) print >> fs_mavg[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) dataIndex['spec_davg_'+i] = 0 dataIndex['spec_dmax_'+i] = 0 dataIndex['spec_dmin_'+i] = 0 dataIndex['spec_mavg_'+i] = 0 js_data.write_begin_data_array(fs_davg[i]) js_data.write_begin_data_array(fs_dmax[i]) js_data.write_begin_data_array(fs_dmin[i]) js_data.write_begin_data_array(fs_mavg[i]) if globals['PLOT_ORIGINAL_TIME_INTERVAL']=='ON': fs_orig[i] = open(output_dir+"data/DSS_compare_spec_orig_"+i+".js",'w') print >> fs_orig[i], """/* Comparison Output File Generated on : %s */"""%(str(Date())) dataIndex['spec_orig_'+i] = 0 js_data.write_begin_data_array(fs_orig[i]) js_data_list.write_begin_data_array(fl) wy_types = ['W','AN','BN','D','C'] if globals['DONOT_SORT_STATION_NAME']=='OFF': output_values = compare_dss_utils.sort_alphabetically(output_values) refname = compare_dss_utils.sort_alphabetically(refname) else: refname = compare_dss_utils.get_not_sort_refname(refname,output_values) for name in refname: ref0 = None; ref1 = None; ref2 = None try: series_name = compare_dss_utils.get_series_name(compare_mode,scalars,refvar[name]) except Exception,e: logging.error("Error building name for :%s and error is %s"%(name,e)) continue if (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='ON' and name in output_values): conti = 0 str_refvar = compare_dss_utils.get_str_refvar(refvar[name],compare_mode) if compare_mode=='1' or compare_mode=='4' or compare_mode=='5': try: if refvar[name][0][4]=='0': dss_group = dss_group0 if refvar[name][0][4]=='1': dss_group = dss_group1 if refvar[name][0][4]=='2': dss_group = dss_group2 ref0 = compare_dss_utils.get_ref(dss_group, refvar[name][0]) p = ref0.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref(ref0) except: print "**** Can't find matched time series for "+ name +" ****" conti = 1 if compare_mode=='3' or compare_mode=='5': try: if refvar[name][2][4]=='0': dss_group = dss_group0 if refvar[name][2][4]=='1': dss_group = dss_group1 if refvar[name][2][4]=='2': dss_group = dss_group2 ref2 = compare_dss_utils.get_ref(dss_group, refvar[name][2]) p = ref2.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref(ref2) except: print "**** Can't find matched time series for "+ name +" ****" conti = 1 if compare_mode=='2' or compare_mode=='3' or compare_mode=='4' or compare_mode=='5': try: if refvar[name][1][4]=='0': dss_group = dss_group0 if refvar[name][1][4]=='1': dss_group = dss_group1 if refvar[name][1][4]=='2': dss_group = dss_group2 ref1 = compare_dss_utils.get_ref(dss_group, refvar[name][1]) p = ref1.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref(ref1) except: print "**** Can't find matched time series for "+ name +" ****" conti = 1 if conti == 0: cpart = p.getPart(p.C_PART).encode('ascii') if name in output_values: ref0_davg = None; ref0_dmax = None; ref0_dmin = None; ref0_mavg = None ref1_davg = None; ref1_dmax = None; ref1_dmin = None; ref1_mavg = None ref2_davg = None; ref2_dmax = None; ref2_dmin = None; ref2_mavg = None if intv < 1000: dataIndex['spec_davg_'+cpart]=dataIndex['spec_davg_'+cpart]+1 dataIndex['spec_dmax_'+cpart]=dataIndex['spec_dmax_'+cpart]+1 dataIndex['spec_dmin_'+cpart]=dataIndex['spec_dmin_'+cpart]+1 if dataIndex['spec_davg_'+cpart]>1: fs_davg[cpart].write(",") if dataIndex['spec_dmax_'+cpart]>1: fs_dmax[cpart].write(",") if dataIndex['spec_dmin_'+cpart]>1: fs_dmin[cpart].write(",") if compare_mode=='1' or compare_mode=='4' or compare_mode=='5': ref0_godin = vtimeseries.godin(ref0) ref0_davg = vtimeseries.per_avg(ref0_godin,'1day') ref0_dmax = vtimeseries.per_max(ref0,'1day') ref0_dmin = vtimeseries.per_min(ref0,'1day') if compare_mode=='2' or compare_mode=='3' or compare_mode=='4' or compare_mode=='5': ref1_godin = vtimeseries.godin(ref1) ref1_davg = vtimeseries.per_avg(ref1_godin,'1day') ref1_dmax = vtimeseries.per_max(ref1,'1day') ref1_dmin = vtimeseries.per_min(ref1,'1day') if compare_mode=='3' or compare_mode=='5': ref2_godin = vtimeseries.godin(ref2) ref2_davg = vtimeseries.per_avg(ref2_godin,'1day') ref2_dmax = vtimeseries.per_max(ref2,'1day') ref2_dmin = vtimeseries.per_min(ref2,'1day') write_plot_data(fs_davg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_davg,ref1_davg,ref2_davg), dataIndex['spec_davg_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Daily Average') write_plot_data(fs_dmax[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_dmax,ref1_dmax,ref2_dmax), dataIndex['spec_dmax_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Daily Maximum') write_plot_data(fs_dmin[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_dmin,ref1_dmin,ref2_dmin), dataIndex['spec_dmin_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Daily Minimum') elif globals['DEFAULT_TIME_INTERVAL']=='1DAY': ref0_davg = ref0; ref0_dmax = ref0; ref0_dmin = ref0 ref1_davg = ref1; ref1_dmax = ref1; ref1_dmin = ref1 ref2_davg = ref2; ref2_dmax = ref2; ref2_dmin = ref2 elif globals['DEFAULT_TIME_INTERVAL']=='1MON': ref2_mavg = ref2; ref2_mmax = ref2; ref2_mmin = ref2 if globals['PLOT_ORIGINAL_TIME_INTERVAL']=='ON': dataIndex['spec_orig_'+cpart]=dataIndex['spec_orig_'+cpart]+1 if dataIndex['spec_orig_'+cpart]>1: fs_orig[cpart].write(",") write_plot_data(fs_orig[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0,ref1,ref2), dataIndex['spec_orig_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Original Time Interval') if intv < 40000: dataIndex['spec_mavg_'+cpart]=dataIndex['spec_mavg_'+cpart]+1 if dataIndex['spec_mavg_'+cpart]>1: fs_mavg[cpart].write(",") if compare_mode=='1' or compare_mode=='4' or compare_mode=='5': ref0_mavg = vtimeseries.per_avg(ref0,'1month') if compare_mode=='2' or compare_mode=='3' or compare_mode=='4' or compare_mode=='5': ref1_mavg = vtimeseries.per_avg(ref1,'1month') if compare_mode=='3' or compare_mode=='5': ref2_mavg = vtimeseries.per_avg(ref2,'1month') write_plot_data(fs_mavg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array(ref0_mavg,ref1_mavg,ref2_mavg), dataIndex['spec_mavg_'+cpart], "%s"%var_name, series_name, "%s (%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES, cpart,'Monthly Average') if (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='ON' and name in output_values): diff_arr = [] if compare_mode=='3' or compare_mode=='4' or compare_mode=='5': for i in range(len(tws)): if compare_mode=='3': rmse_val = vdiff.rmse(ref1, ref2, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse(ref1, ref2, tws[i], time_interval) diff_arr.append([perc_rmse_val,rmse_val ]) if compare_mode=='4': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse(ref0, ref1, tws[i], time_interval) diff_arr.append([perc_rmse_val,rmse_val ]) if compare_mode=='5': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse(ref0, ref1, tws[i], time_interval) rmse_val2 = vdiff.rmse(ref0, ref2, tws[i], time_interval) perc_rmse_val2 = vdiff.perc_rmse(ref0, ref2, tws[i], time_interval) diff_arr.append([perc_rmse_val,rmse_val,perc_rmse_val2,rmse_val2 ]) for w in wy_types: if compare_mode=='3': diff_arr.append([vdiff.rmse_discrete_tws(ref1,ref2,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref1,ref2,wyt_arr[w],1)]) if compare_mode=='4': diff_arr.append([vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],1)]) if compare_mode=='5': diff_arr.append([vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref0,ref1,wyt_arr[w],1), vdiff.rmse_discrete_tws(ref0,ref2,wyt_arr[w],0), vdiff.rmse_discrete_tws(ref0,ref2,wyt_arr[w],1)]) latlng = compare_dss_utils.get_latlng(tbl_latlng,p.getPart(p.B_PART)) if globals['CALCULATE_SPECIFIED_RMSE_ONLY']=='OFF': logging.debug('Working on index: %d/%d '%(dIndex+1,len(refname))) else: logging.debug('Working on index: %d/%d '%(dIndex+1,len(output_values))) dIndex = dIndex + 1 if latlng==None: latlng=['nan','nan','nan','nan'] print "can't find the Lat/Lng for ",p.getPart(p.B_PART) if dIndex>1: fl.write(",") if name in output_values: write_list_data(fl,name, str_refvar, p.getPart(p.B_PART), cpart, 1, diff_arr,latlng) else: write_list_data(fl,name, str_refvar, p.getPart(p.B_PART), cpart, 0, diff_arr,latlng) else: logging.debug("*** Please verify the path for "+name+" ***")
def do_processing(globals, scalars, var_values, output_values, tw_values): compare_mode = globals['COMPARE_MODE'] time_interval = globals['DEFAULT_TIME_INTERVAL'] # open files 1 and file 2 and loop over to plot from java.util import Date dss_group0, dss_group1, dss_group2, refvar, refname = compare_dss_utils.get_group_ref( globals, scalars, var_values) # obtain the new output list modified for *_ or _* cases output_values = compare_dss_utils.get_ref_list(compare_mode, dss_group0, dss_group1, output_values) output_dir = scalars['OUTDIR'].replace('"', '') wyt_arr = write_wyt(output_dir, tw_values) #write the water type JavaScript file #out_name = column(output_values,0) #out_type = column(output_values,1) if compare_mode == '1': type_arr = compare_dss_utils.get_cpart_list(dss_group0) else: type_arr = compare_dss_utils.get_cpart_list(dss_group1) compare_dss_utils.chk_cpart_space(type_arr) if output_dir[-1] != '/': output_dir = output_dir + "/" time_windows = map(lambda val: val[1].replace('"', ''), tw_values) tws = map(lambda x: vtimeseries.timewindow(x), time_windows) dIndex = 0 dataIndex = {} fs_davg = {} fs_dmax = {} fs_dmin = {} fs_mavg = {} fs_orig = {} tbl_latlng = compare_dss_utils.lines2table(output_dir + "js/latlng.txt") fl = open(output_dir + "data/data_list.js", 'w') for i in type_arr: initial_js = output_dir + "data/DSS_compare_spec_davg_" + i + ".js" initial_pretab = i fs_davg[i] = open( output_dir + "data/DSS_compare_spec_davg_" + i + ".js", 'w') fs_dmax[i] = open( output_dir + "data/DSS_compare_spec_dmax_" + i + ".js", 'w') fs_dmin[i] = open( output_dir + "data/DSS_compare_spec_dmin_" + i + ".js", 'w') fs_mavg[i] = open( output_dir + "data/DSS_compare_spec_mavg_" + i + ".js", 'w') print >> fs_davg[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) print >> fs_dmax[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) print >> fs_dmin[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) print >> fs_mavg[ i], """/* Comparison Output File Generated on : %s */""" % (str( Date())) dataIndex['spec_davg_' + i] = 0 dataIndex['spec_dmax_' + i] = 0 dataIndex['spec_dmin_' + i] = 0 dataIndex['spec_mavg_' + i] = 0 js_data.write_begin_data_array(fs_davg[i]) js_data.write_begin_data_array(fs_dmax[i]) js_data.write_begin_data_array(fs_dmin[i]) js_data.write_begin_data_array(fs_mavg[i]) if globals['PLOT_ORIGINAL_TIME_INTERVAL'] == 'ON': fs_orig[i] = open( output_dir + "data/DSS_compare_spec_orig_" + i + ".js", 'w') print >> fs_orig[ i], """/* Comparison Output File Generated on : %s */""" % ( str(Date())) dataIndex['spec_orig_' + i] = 0 js_data.write_begin_data_array(fs_orig[i]) js_data_list.write_begin_data_array(fl) wy_types = ['W', 'AN', 'BN', 'D', 'C'] if globals['DONOT_SORT_STATION_NAME'] == 'OFF': output_values = compare_dss_utils.sort_alphabetically(output_values) refname = compare_dss_utils.sort_alphabetically(refname) else: refname = compare_dss_utils.get_not_sort_refname( refname, output_values) for name in refname: ref0 = None ref1 = None ref2 = None try: series_name = compare_dss_utils.get_series_name( compare_mode, scalars, refvar[name]) except Exception, e: logging.error("Error building name for :%s and error is %s" % (name, e)) continue if (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'ON' and name in output_values): conti = 0 str_refvar = compare_dss_utils.get_str_refvar( refvar[name], compare_mode) if compare_mode == '1' or compare_mode == '4' or compare_mode == '5': try: if refvar[name][0][4] == '0': dss_group = dss_group0 if refvar[name][0][4] == '1': dss_group = dss_group1 if refvar[name][0][4] == '2': dss_group = dss_group2 ref0 = compare_dss_utils.get_ref(dss_group, refvar[name][0]) p = ref0.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref( ref0) except: print "**** Can't find matched time series for " + name + " ****" conti = 1 if compare_mode == '3' or compare_mode == '5': try: if refvar[name][2][4] == '0': dss_group = dss_group0 if refvar[name][2][4] == '1': dss_group = dss_group1 if refvar[name][2][4] == '2': dss_group = dss_group2 ref2 = compare_dss_utils.get_ref(dss_group, refvar[name][2]) p = ref2.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref( ref2) except: print "**** Can't find matched time series for " + name + " ****" conti = 1 if compare_mode == '2' or compare_mode == '3' or compare_mode == '4' or compare_mode == '5': try: if refvar[name][1][4] == '0': dss_group = dss_group0 if refvar[name][1][4] == '1': dss_group = dss_group1 if refvar[name][1][4] == '2': dss_group = dss_group2 ref1 = compare_dss_utils.get_ref(dss_group, refvar[name][1]) p = ref1.pathname intv, var_name, data_units, data_type = compare_dss_utils.get_prop_of_ref( ref1) except: print "**** Can't find matched time series for " + name + " ****" conti = 1 if conti == 0: cpart = p.getPart(p.C_PART).encode('ascii') if name in output_values: ref0_davg = None ref0_dmax = None ref0_dmin = None ref0_mavg = None ref1_davg = None ref1_dmax = None ref1_dmin = None ref1_mavg = None ref2_davg = None ref2_dmax = None ref2_dmin = None ref2_mavg = None if intv < 1000: dataIndex['spec_davg_' + cpart] = dataIndex['spec_davg_' + cpart] + 1 dataIndex['spec_dmax_' + cpart] = dataIndex['spec_dmax_' + cpart] + 1 dataIndex['spec_dmin_' + cpart] = dataIndex['spec_dmin_' + cpart] + 1 if dataIndex['spec_davg_' + cpart] > 1: fs_davg[cpart].write(",") if dataIndex['spec_dmax_' + cpart] > 1: fs_dmax[cpart].write(",") if dataIndex['spec_dmin_' + cpart] > 1: fs_dmin[cpart].write(",") if compare_mode == '1' or compare_mode == '4' or compare_mode == '5': ref0_godin = vtimeseries.godin(ref0) ref0_davg = vtimeseries.per_avg(ref0_godin, '1day') ref0_dmax = vtimeseries.per_max(ref0, '1day') ref0_dmin = vtimeseries.per_min(ref0, '1day') if compare_mode == '2' or compare_mode == '3' or compare_mode == '4' or compare_mode == '5': ref1_godin = vtimeseries.godin(ref1) ref1_davg = vtimeseries.per_avg(ref1_godin, '1day') ref1_dmax = vtimeseries.per_max(ref1, '1day') ref1_dmin = vtimeseries.per_min(ref1, '1day') if compare_mode == '3' or compare_mode == '5': ref2_godin = vtimeseries.godin(ref2) ref2_davg = vtimeseries.per_avg(ref2_godin, '1day') ref2_dmax = vtimeseries.per_max(ref2, '1day') ref2_dmin = vtimeseries.per_min(ref2, '1day') write_plot_data( fs_davg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_davg, ref1_davg, ref2_davg), dataIndex['spec_davg_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Daily Average') write_plot_data( fs_dmax[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_dmax, ref1_dmax, ref2_dmax), dataIndex['spec_dmax_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Daily Maximum') write_plot_data( fs_dmin[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_dmin, ref1_dmin, ref2_dmin), dataIndex['spec_dmin_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Daily Minimum') elif globals['DEFAULT_TIME_INTERVAL'] == '1DAY': ref0_davg = ref0 ref0_dmax = ref0 ref0_dmin = ref0 ref1_davg = ref1 ref1_dmax = ref1 ref1_dmin = ref1 ref2_davg = ref2 ref2_dmax = ref2 ref2_dmin = ref2 elif globals['DEFAULT_TIME_INTERVAL'] == '1MON': ref2_mavg = ref2 ref2_mmax = ref2 ref2_mmin = ref2 if globals['PLOT_ORIGINAL_TIME_INTERVAL'] == 'ON': dataIndex['spec_orig_' + cpart] = dataIndex['spec_orig_' + cpart] + 1 if dataIndex['spec_orig_' + cpart] > 1: fs_orig[cpart].write(",") write_plot_data( fs_orig[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0, ref1, ref2), dataIndex['spec_orig_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Original Time Interval') if intv < 40000: dataIndex['spec_mavg_' + cpart] = dataIndex['spec_mavg_' + cpart] + 1 if dataIndex['spec_mavg_' + cpart] > 1: fs_mavg[cpart].write(",") if compare_mode == '1' or compare_mode == '4' or compare_mode == '5': ref0_mavg = vtimeseries.per_avg(ref0, '1month') if compare_mode == '2' or compare_mode == '3' or compare_mode == '4' or compare_mode == '5': ref1_mavg = vtimeseries.per_avg(ref1, '1month') if compare_mode == '3' or compare_mode == '5': ref2_mavg = vtimeseries.per_avg(ref2, '1month') write_plot_data( fs_mavg[cpart], compare_mode, name, str_refvar, compare_dss_utils.build_data_array( ref0_mavg, ref1_mavg, ref2_mavg), dataIndex['spec_mavg_' + cpart], "%s" % var_name, series_name, "%s (%s)" % (data_type, data_units), "Time", PlotType.TIME_SERIES, cpart, 'Monthly Average') if (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'OFF') or (globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'ON' and name in output_values): diff_arr = [] if compare_mode == '3' or compare_mode == '4' or compare_mode == '5': for i in range(len(tws)): if compare_mode == '3': rmse_val = vdiff.rmse(ref1, ref2, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse( ref1, ref2, tws[i], time_interval) diff_arr.append([perc_rmse_val, rmse_val]) if compare_mode == '4': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse( ref0, ref1, tws[i], time_interval) diff_arr.append([perc_rmse_val, rmse_val]) if compare_mode == '5': rmse_val = vdiff.rmse(ref0, ref1, tws[i], time_interval) perc_rmse_val = vdiff.perc_rmse( ref0, ref1, tws[i], time_interval) rmse_val2 = vdiff.rmse(ref0, ref2, tws[i], time_interval) perc_rmse_val2 = vdiff.perc_rmse( ref0, ref2, tws[i], time_interval) diff_arr.append([ perc_rmse_val, rmse_val, perc_rmse_val2, rmse_val2 ]) for w in wy_types: if compare_mode == '3': diff_arr.append([ vdiff.rmse_discrete_tws( ref1, ref2, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref1, ref2, wyt_arr[w], 1) ]) if compare_mode == '4': diff_arr.append([ vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 1) ]) if compare_mode == '5': diff_arr.append([ vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref0, ref1, wyt_arr[w], 1), vdiff.rmse_discrete_tws( ref0, ref2, wyt_arr[w], 0), vdiff.rmse_discrete_tws( ref0, ref2, wyt_arr[w], 1) ]) latlng = compare_dss_utils.get_latlng( tbl_latlng, p.getPart(p.B_PART)) if globals['CALCULATE_SPECIFIED_RMSE_ONLY'] == 'OFF': logging.debug('Working on index: %d/%d ' % (dIndex + 1, len(refname))) else: logging.debug('Working on index: %d/%d ' % (dIndex + 1, len(output_values))) dIndex = dIndex + 1 if latlng == None: latlng = ['nan', 'nan', 'nan', 'nan'] print "can't find the Lat/Lng for ", p.getPart( p.B_PART) if dIndex > 1: fl.write(",") if name in output_values: write_list_data(fl, name, str_refvar, p.getPart(p.B_PART), cpart, 1, diff_arr, latlng) else: write_list_data(fl, name, str_refvar, p.getPart(p.B_PART), cpart, 0, diff_arr, latlng) else: logging.debug("*** Please verify the path for " + name + " ***")