def _test2():
append=0
for fpart in ['dxc-op','dxc-cl']:
g1 = vutils.opendss('anninputs.dss')
g2 = vutils.opendss('annoutputs.dss')
g1.filterBy(fpart)
refs = g1[:]
tw = vutils.timewindow('01oct1975 0000 - 01sep1991 0000')
#tw = vutils.timewindow('01oct1975 0000 - 01oct1976 0000')
inps = []
for i in range(len(refs)):
inps.append(vutils.interpolate(DataReference.create(refs[i],tw).getData(),'1day'))
inputs = []
for i in range(len(inps)):
print 'Building inputs for ',inps[i]
inputs=inputs+buildinput(inps[i],7,10,7) # weekly averages
print 'Built inputs'
outputs = []
ccc_ref = vutils.findpath(g2,'//ccc/ec///%s/'%fpart)[0]
outputs.append(vutils.interpolate(DataReference.create(ccc_ref,tw).getData(),'1day'))
print 'Built outputs'
print 'Dumping patterns'
if append:
dump_patterns(inputs,outputs,'junk',0.75,365,1)
else:
dump_patterns(inputs,outputs,'junk',0.75,365,append)
append = 1
def compare_dss_files(file1, file2, showPlot=False, outputFile=None, outputPathFile=None):
"""
Simply compares the files and outputs differences if any of those that differ and lists mismatching pathnames in either
"""
g1 = vutils.opendss(file1)
g2 = vutils.opendss(file2)
print 'Comparing %s to %s' % (file1, file2)
print '%12s\t%32s' % ('DIFFERENCE', 'PATHNAME')
if outputPathFile:
opf_handle=open(outputPathFile, 'wb')
opf = csv.writer(opf_handle, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
opf.writerow([file1,file2])
no_diff=True
for ref1 in g1:
p1 = ref1.pathname
found = False
for ref2 in g2:
p2 = ref2.pathname
if matches(p1, p2):
found = True
diff = ref2.data - ref1.data
absdiff = diff.createSlice(diff.getTimeWindow())
vtimeseries.apply(absdiff, math.fabs)
diff_total = vtimeseries.total(absdiff)
if (diff_total > 1e-06) :
no_diff=False
if showPlot: plot(ref1.data, ref2.data)
print '%10.2f\t%32s' % (diff_total, p1)
if outputFile:
diffp = set_part(p1, 'DIFF-%s-%s' % (os.path.basename(file1), os.path.basename(file2)), Pathname.A_PART)
writedss(outputFile, str(diffp), diff)
if outputPathFile:
opf.writerow([p1, p1, diff_total])
break
if (not found):
no_diff=False
print 'No matching path: %s in file %s NOT found in file %s' % (p1, file1, file2)
if outputPathFile: opf.writerow([p1, "", "M"])
for ref2 in g2:
p2 = ref2.pathname
found = False
for ref1 in g1:
p1 = ref1.pathname
if matches(p1, p2):
found = True
break
if (not found):
no_diff=False
print 'No matching path: %s in file %s NOT found in file %s' % (p2, file2, file1)
if outputPathFile: opf.writerow(["", p2, "M"])
if no_diff:
print 'NO DIFFERENCE ACROSS ENTIRE FILEs %s and %s'%(file1,file2)
if outputPathFile: opf_handle.close()
def _test3():
from ANN import fnet_cccec
fpart = 'dxc-op'
g1 = vutils.opendss('anninputs.dss')
g2 = vutils.opendss('annoutputs.dss')
g1.filterBy(fpart)
g2.filterBy(fpart)
refs = g1[:]
tw = vutils.timewindow('01oct1975 0000 - 01sep1991 0000')
#tw = vutils.timewindow('01oct1975 0000 - 01oct1976 0000')
inps = []
for i in range(len(refs)):
inps.append(vutils.interpolate(DataReference.create(refs[i],tw).getData(),'1day'))
inputs = []
for i in range(len(inps)):
print 'Building inputs for ',inps[i]
inputs=inputs+buildinput(inps[i],7,10,7) # weekly averages
print 'Built inputs'
outputs = []
ccc_ref = vutils.findpath(g2,'//ccc/ec///%s/'%fpart)[0]
outputs.append(vutils.interpolate(DataReference.create(ccc_ref,tw).getData(),'1day'))
mi = MultiIterator(inputs)
import jarray
ninps = len(inputs)
input = jarray.zeros(ninps,'f')
output = jarray.zeros(1,'f')
ann = fnet_cccec()
ndata = len(inputs[0])
for input_no in range(365):
mi.advance()
outdata = jarray.zeros(ndata,'f')
while input_no < ndata:
el = mi.getElement()
i=0
while i < ninps:
input[i] = el.getY(i)
i=i+1
ann.engine(input,output,0)
outdata[input_no] = output[0]
mi.advance()
input_no=input_no+1
#
stime = inputs[0].getStartTime()
ti = inputs[0].getTimeInterval()
rtsout = vutils.RegularTimeSeries('/ann/ccc_out/ec///annutils/',str(stime),\
str(ti),outdata)
vutils.plot(rtsout,outputs[0])
rtsout = (rtsout-0.140516)/0.000396563
vutils.writedss('annout.dss','/ann/ccc_out/ec///annutils/',rtsout)
def testspline():
from vutils import opendss, findpath, timewindow, tabulate
infile='/delta/data/dss/dayflo.dss'
inpath='/DELTA/OUT/FLOW//1day//'
tw_str='01JAN1999 0000 - 30JUN1999 2400'
g=opendss(infile)
ref = findpath(g,inpath)[0]
ref = DataReference.create(ref,timewindow(tw_str))
rts=interpolate(ref,outint='15min',offset=48)
tabulate(rts)
def testlinear():
from vutils import opendss, findpath, timewindow, tabulate
infile='h:/data/dss/IEP/hydro.dss'
inpath='/RLTM\+CHAN/RSAC155/FLOW//1HOUR/CDEC/'
tw_str='16JAN1999 1200 - 21JAN1999 1600'
g=opendss(infile)
ref = findpath(g,inpath)[0]
ref = DataReference.create(ref,timewindow(tw_str))
rts=interplinear(ref)
tabulate(rts,ref.getData())
def rename(file,oldf,newfile,newf):
g=vutils.opendss(file)
g.filterBy(PathPartPredicate(oldf,Pathname.F_PART),1)
if len(g) == 0:
raise SystemExit,'No FPART: %s in DSS FILE: %s'%(oldf,file)
for ref in g:
path = ref.getPathname()
ds = ref.getData()
path.setPart(Pathname.F_PART,newf)
print 'OLD: %s -> NEW: %s'%(str(ref.getPathname()),str(path))
vutils.writedss(newfile,str(path),ds)
g.removeDataReference(ref)
def rename(file, oldf, newfile, newf):
g = vutils.opendss(file)
g.filterBy(PathPartPredicate(oldf, Pathname.F_PART), 1)
if len(g) == 0:
raise SystemExit, 'No FPART: %s in DSS FILE: %s' % (oldf, file)
for ref in g:
path = ref.getPathname()
ds = ref.getData()
path.setPart(Pathname.F_PART, newf)
print 'OLD: %s -> NEW: %s' % (str(ref.getPathname()), str(path))
vutils.writedss(newfile, str(path), ds)
g.removeDataReference(ref)
def _test1():
g=vutils.opendss('/delta2/ann/hydrologies/sim809anndv.dss')
refs = vutils.findpath(g,'//ndo/flow-ndo///')
tw = vutils.timewindow('01jan1974 0000 - 01jan1992 0000')
inps = []
for i in range(len(refs)):
inps.append(vutils.interpolate(DataReference.create(refs[i],tw).getData(),'1day'))
print 'Got data for ',tw
scaling = 8.99649e-07
shift = 0.233677
crush(inps[0],scaling,shift)
print 'Crushed data'
inputs = buildinput(inps[0],8,10,7)
print 'Built inputs'
outputs = inps[:]
print 'Dumping patterns'
dump_patterns(inputs,outputs,'junk',0.75)
def do_processing(scalars, pathname_maps, tw_values):
# open files 1 and file 2 and loop over to plot
dss_group1 = vutils.opendss(scalars['FILE1'])
dss_group2 = vutils.opendss(scalars['FILE2'])
time_windows = map(lambda val: val[1].replace('"',''), tw_values)
tws = map(lambda x: vtimeseries.timewindow(x), time_windows)
if len(tws) > 0:
tw=tws[0]
else:
tw=None
output_file=scalars['OUTFILE']
data_output_file = output_file.split(".")[0]+".js"
fh=open(data_output_file,'w')
print >> fh, """/*
Comparison Output File
Generated on : %s
*/"""%(str(Date()))
js_data.write_begin_data_array(fh);
if dss_group1 == None or dss_group2 == None:
sys.exit(2);
dataIndex=0
for path_map in pathname_maps:
dataIndex=dataIndex+1
logging.debug('Working on index: %d'%dataIndex)
if dataIndex>1:
fh.write(",")
#path_map = pathname_mapping[var_name]
if path_map.path2==None or path_map.path2 == "":
path_map.path2=path_map.path1
var_name = path_map.var_name
calculate_dts=0
if path_map.var_category == 'HEADER':
logging.debug('Inserting header')
continue;
if path_map.report_type == 'Exceedance_Post':
calculate_dts=1
ref1 = get_ref(dss_group1, path_map.path1,calculate_dts, pathname_maps, 1)
ref2 = get_ref(dss_group2, path_map.path2,calculate_dts, pathname_maps, 2)
if (ref1==None or ref2==None):
continue
series_name = [scalars['NAME1'],scalars['NAME2']]
data_units=get_units(ref1,ref2)
data_type=get_type(ref1,ref2)
if path_map.report_type == 'Average':
write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES)
elif path_map.report_type == 'Exceedance':
write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE)
elif path_map.report_type == 'Avg_Excd':
write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES)
fh.write(",")
write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE)
elif path_map.report_type == 'Timeseries':
write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES)
elif path_map.report_type == 'Exceedance_Post':
write_plot_data(fh, build_exceedance_array(ref1,ref2,tw), dataIndex, "Exceedance %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE)
js_data.write_end_data_array(fh);
logging.debug('Writing end of data array')
fh.close()
# Generate the main html file
fh=open(scalars['OUTFILE'],'w')
print >> fh, """
<html>
<head>
<title>Calsim Report: %s vs %s</title>
<script type="text/javascript" src="%s"></script>
<script type="text/javascript" src="report-common/protovis-d3.3.js"></script>
<script type="text/javascript" src="report-common/plots.js"></script>
<script type="text/javascript" src="report-common/jquery-1.4.2.min.js"></script>
<link rel="stylesheet" type="text/css" media="print" href="report-common/print.css" />
<link rel="stylesheet" type="text/css" media="screen" href="report-common/screen.css" />
</head>
"""%(scalars['NAME1'],scalars['NAME2'],data_output_file)
write_control_panel(fh,scalars, pathname_maps,tw_values)
write_water_balance_table(fh,dss_group1, dss_group2, scalars, pathname_maps,tw_values)
print >> fh, """<script type="text/javascript">
function clear_and_draw(sdate, edate){
$('.plot').empty();
n=data.length
plot_diff = $('input[name=diff_plot]').is(':checked') ? 1 : 0 ;
for(i=0; i < n; i++){
var div_id = "fig"+(i+1);
if (data[i]==null) continue;
if ($("#"+div_id).length==0){
$("body").append('<a href="#'+div_id+'"><div class="plot" id="'+div_id+'"></div></a>');
}
if (data[i].plot_type=="timeseries"){
plots.time_series_plot(div_id,data[i],plot_diff,sdate,edate);
}else if (data[i].plot_type=="exceedance"){
plots.exceedance_plot(div_id,data[i],null,sdate,edate);
}
}
// update links
};
$(document).ready(clear_and_draw(null,null));
$('#system-water-balance-table tr').click(function(){
var_name = $($(this).find('td')[0]).text();
var svg_element = $('div').filter(function(){ return $(this).text().indexOf(var_name)>=0;})
if (svg_element && svg_element.length > 0){
anchor_name = $($(svg_element[0]).parent()).attr("href");
window.location.href=window.location.href.split('#')[0]+anchor_name;
}
});
</script>
<script type="text/javascript">
function extract_date(date_str){
date_fields=date_str.split(",");
return new Date(date_fields[0],date_fields[1],date_fields[2]);
}
$('#time-window-select').change(function(){
var changed_val = $('#time-window-select option:selected').val().split("-");
clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1]));
});
$('#threshold').change(function(){
set_diff_threshold($('#threshold').val());
});
$('input[name=diff_plot]').change(function(){
var changed_val = $('#time-window-select option:selected').val().split("-");
clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1]));
});
function set_diff_threshold(threshold){
var diff_marker = function(){
if(Math.abs(parseFloat($(this).text())) >= threshold){
$(this).addClass('large-diff');
}else{
$(this).removeClass('large-diff');
};
};
$('tr td:nth-child(5)').each(diff_marker);
$('tr td:nth-child(9)').each(diff_marker);
$('tr td:nth-child(13)').each(diff_marker);
};
set_diff_threshold($('#threshold').val());
</script>
</body>
</html>"""
fh.close()
logging.debug('Closed out data file')
def open_dss(dssfile):
group = vutils.opendss(dssfile)
if group == None:
print "No dss file named: %s found!" % (dssfile)
return group
def get_group_ref(globals, scalars, var_values):
refvar = {}
refnam = []
compare_mode = globals['COMPARE_MODE']
dss_group0 = 'NA'
dss_group1 = 'NA'
dss_group2 = 'NA'
try:
dss_group0 = vutils.opendss(scalars['FILE0'])
dss_name0 = scalars['NAME0']
for ref0 in dss_group0:
p = ref0.pathname
bpart = p.getPart(p.B_PART).encode('ascii')
cpart = p.getPart(p.C_PART).encode('ascii')
epart = p.getPart(p.E_PART).encode('ascii')
if epart==globals['DEFAULT_TIME_INTERVAL']:
refnam.append(bpart+'_'+cpart)
refvar[bpart+'_'+cpart]=['FILE0:://'+bpart+'/'+cpart+'//'+epart+'//','NA','NA']
except:
if (compare_mode=='1' or compare_mode=='4' or compare_mode=='5'):
print "**** Please specify observation file for this comparison mode!!! ****"
try:
dss_group1 = vutils.opendss(scalars['FILE1'])
dss_name1 = scalars['NAME1']
for ref1 in dss_group1:
p = ref1.pathname
bpart = p.getPart(p.B_PART).encode('ascii')
cpart = p.getPart(p.C_PART).encode('ascii')
epart = p.getPart(p.E_PART).encode('ascii')
if epart==globals['DEFAULT_TIME_INTERVAL']:
refnam.append(bpart+'_'+cpart)
try:
refvar[bpart+'_'+cpart]
refvar[bpart+'_'+cpart][1]='FILE1:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
refvar[bpart+'_'+cpart]=['NA','NA','NA']
refvar[bpart+'_'+cpart][1]='FILE1:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
if compare_mode!='0':
print "**** Please specify model dss file 1 for this comparison mode!!! ****"
try:
dss_group2 = vutils.opendss(scalars['FILE2'])
dss_name2 = scalars['NAME2']
for ref2 in dss_group2:
p = ref2.pathname
bpart = p.getPart(p.B_PART).encode('ascii')
cpart = p.getPart(p.C_PART).encode('ascii')
epart = p.getPart(p.E_PART).encode('ascii')
if epart==globals['DEFAULT_TIME_INTERVAL']:
refnam.append(bpart+'_'+cpart)
try:
refvar[bpart+'_'+cpart]
refvar[bpart+'_'+cpart][2]='FILE2:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
refvar[bpart+'_'+cpart]=['NA','NA','NA']
refvar[bpart+'_'+cpart][2]='FILE2:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
if compare_mode=='3' or compare_mode=='5':
print "**** Please specify model dss file 2 for this comparison mode!!! ****"
'''delete time series without a matched entry'''
for name in refnam:
try:
if (compare_mode=='1' and refvar[name][0]=='NA') or \
(compare_mode=='2' and refvar[name][1]=='NA') or \
(compare_mode=='3' and (refvar[name][1]=='NA' or refvar[name][2]=='NA')) or \
(compare_mode=='4' and (refvar[name][0]=='NA' or refvar[name][1]=='NA')) or \
(compare_mode=='5' and (refvar[name][0]=='NA' or refvar[name][1]=='NA' or refvar[name][2]=='NA')):
del(refvar[name])
refnam.remove(name)
except Exception,e:
refnam.remove(name)
logging.error("Error building name for %s"%(name))
continue
def get_group_ref(globals, scalars, var_values):
refvar = {}
refnam = []
compare_mode = globals['COMPARE_MODE']
dss_group0 = 'NA'
dss_group1 = 'NA'
dss_group2 = 'NA'
try:
dss_group0 = vutils.opendss(scalars['FILE0'])
dss_name0 = scalars['NAME0']
for ref0 in dss_group0:
p = ref0.pathname
bpart = p.getPart(p.B_PART).encode('ascii')
cpart = p.getPart(p.C_PART).encode('ascii')
epart = p.getPart(p.E_PART).encode('ascii')
if epart==globals['DEFAULT_TIME_INTERVAL']:
refnam.append(bpart+'_'+cpart)
refvar[bpart+'_'+cpart]=['FILE0:://'+bpart+'/'+cpart+'//'+epart+'//','NA','NA']
except:
if (compare_mode=='1' or compare_mode=='4' or compare_mode=='5'):
print "**** Please specify observation file for this comparison mode!!! ****"
try:
dss_group1 = vutils.opendss(scalars['FILE1'])
dss_name1 = scalars['NAME1']
for ref1 in dss_group1:
p = ref1.pathname
bpart = p.getPart(p.B_PART).encode('ascii')
cpart = p.getPart(p.C_PART).encode('ascii')
epart = p.getPart(p.E_PART).encode('ascii')
if epart==globals['DEFAULT_TIME_INTERVAL']:
refnam.append(bpart+'_'+cpart)
try:
refvar[bpart+'_'+cpart]
refvar[bpart+'_'+cpart][1]='FILE1:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
refvar[bpart+'_'+cpart]=['NA','NA','NA']
refvar[bpart+'_'+cpart][1]='FILE1:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
if compare_mode!='0':
print "**** Please specify model dss file 1 for this comparison mode!!! ****"
try:
dss_group2 = vutils.opendss(scalars['FILE2'])
dss_name2 = scalars['NAME2']
for ref2 in dss_group2:
p = ref2.pathname
bpart = p.getPart(p.B_PART).encode('ascii')
cpart = p.getPart(p.C_PART).encode('ascii')
epart = p.getPart(p.E_PART).encode('ascii')
if epart==globals['DEFAULT_TIME_INTERVAL']:
refnam.append(bpart+'_'+cpart)
try:
refvar[bpart+'_'+cpart]
refvar[bpart+'_'+cpart][2]='FILE2:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
refvar[bpart+'_'+cpart]=['NA','NA','NA']
refvar[bpart+'_'+cpart][2]='FILE2:://'+bpart+'/'+cpart+'//'+epart+'//'
except:
if compare_mode=='3' or compare_mode=='5':
print "**** Please specify model dss file 2 for this comparison mode!!! ****"
'''delete time series without a matched entry'''
for name in refnam:
try:
if (compare_mode=='1' and refvar[name][0]=='NA') or \
(compare_mode=='2' and refvar[name][1]=='NA') or \
(compare_mode=='3' and (refvar[name][1]=='NA' or refvar[name][2]=='NA')) or \
(compare_mode=='4' and (refvar[name][0]=='NA' or refvar[name][1]=='NA')) or \
(compare_mode=='5' and (refvar[name][0]=='NA' or refvar[name][1]=='NA' or refvar[name][2]=='NA')):
del(refvar[name])
refnam.remove(name)
except Exception,e:
refnam.remove(name)
logging.error("Error building name for %s"%(name))
continue
from vtimeseries import time
NaN=float('nan')
def export_csv(ref,file):
d = ref.data
f=open(file,'w')
for e in d:
f.write("%s,%s\n"%(e.x, "" if e.y == -901.0 or e.y == -902.0 else e.y))
f.close()
if __name__=='__main__':
file='Z:/DSM2_v81_Beta_Release/studies/historical_qual_ec_v81/output/historical_v81.dss'
from vutils import opendss
import vdss
ref = vdss.findpath(opendss(file),'//sltrm004/flow')[0]
print 'Exporting ref: %s'%ref
export_csv(ref,'Z:/temp/sltrm004_flow.csv')
import vutils
import datetime
if __name__ == '__main__':
print('Uses Vista which wraps lowlevel HEC DSSVue functions')
d = vutils.opendss('ITP_PP_out_ec.dss')
s = datetime.datetime.now()
npaths = len(d) # Used to load the catalog into memory
print('Catalog read in %s' % str(datetime.datetime.now() - s))
print('Reading data from %s pathnames' % str(npaths))
s = datetime.datetime.now()
for ref in d:
s1 = datetime.datetime.now()
data = ref.getData()
path = ref.getPathname()
print('Read %s in %s ' %
(str(path), str(datetime.datetime.now() - s1)))
print('Read %s in %s' % (npaths, str(datetime.datetime.now() - s)))
def open_dss(dssfile):
group = vutils.opendss(dssfile)
if group == None:
print "No dss file named: %s found!" % (dssfile)
return group
def do_processing(scalars, pathname_maps, tw_values):
# open files 1 and file 2 and loop over to plot
dss_group1 = vutils.opendss(scalars['FILE1'])
dss_group2 = vutils.opendss(scalars['FILE2'])
time_windows = map(lambda val: val[1].replace('"',''), tw_values)
tws = map(lambda x: vtimeseries.timewindow(x), time_windows)
if len(tws) > 0:
tw=tws[0]
else:
tw=None
output_file=scalars['OUTFILE']
data_output_file = output_file.split(".")[0]+".js"
fh=open(data_output_file,'w')
print >> fh, """/*
Comparison Output File
Generated on : %s
*/"""%(str(Date()))
js_data.write_begin_data_array(fh);
if dss_group1 == None or dss_group2 == None:
sys.exit(2);
dataIndex=0
for path_map in pathname_maps:
dataIndex=dataIndex+1
logging.debug('Working on index: %d'%dataIndex)
if dataIndex>1:
fh.write(",")
#path_map = pathname_mapping[var_name]
if path_map.path2==None or path_map.path2 == "":
path_map.path2=path_map.path1
var_name = path_map.var_name
calculate_dts=0
if path_map.var_category == 'HEADER':
logging.debug('Inserting header')
continue;
if path_map.report_type == 'Exceedance_Post':
calculate_dts=1
ref1 = get_ref(dss_group1, path_map.path1,calculate_dts, pathname_maps, 1)
ref2 = get_ref(dss_group2, path_map.path2,calculate_dts, pathname_maps, 2)
if (ref1==None or ref2==None):
continue
series_name = [scalars['NAME1'],scalars['NAME2']]
data_units=get_units(ref1,ref2)
data_type=get_type(ref1,ref2)
if path_map.report_type == 'Average':
write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES)
elif path_map.report_type == 'Exceedance':
write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE)
elif path_map.report_type == 'Avg_Excd':
write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES)
fh.write(",")
write_plot_data(fh, build_exceedance_array(ref1,ref2,path_map.var_category=='S_SEPT',tw), dataIndex, get_exceedance_plot_title(path_map), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE)
elif path_map.report_type == 'Timeseries':
write_plot_data(fh, build_data_array(ref1,ref2,tw), dataIndex, "Average %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Time", PlotType.TIME_SERIES)
elif path_map.report_type == 'Exceedance_Post':
write_plot_data(fh, build_exceedance_array(ref1,ref2,tw), dataIndex, "Exceedance %s"%path_map.var_name.replace('"',''), series_name, "%s(%s)"%(data_type,data_units), "Percent at or above", PlotType.EXCEEDANCE)
js_data.write_end_data_array(fh);
logging.debug('Writing end of data array')
fh.close()
# Generate the main html file
fh=open(scalars['OUTFILE'],'w')
print >> fh, """
<html>
<head>
<title>Calsim Report: %s vs %s</title>
<script type="text/javascript" src="%s"></script>
<script type="text/javascript" src="report-common/protovis-d3.3.js"></script>
<script type="text/javascript" src="report-common/plots.js"></script>
<script type="text/javascript" src="report-common/jquery-1.4.2.min.js"></script>
<link rel="stylesheet" type="text/css" media="print" href="report-common/print.css" />
<link rel="stylesheet" type="text/css" media="screen" href="report-common/screen.css" />
</head>
"""%(scalars['NAME1'],scalars['NAME2'],data_output_file)
write_control_panel(fh,scalars, pathname_maps,tw_values)
write_water_balance_table(fh,dss_group1, dss_group2, scalars, pathname_maps,tw_values)
print >> fh, """<script type="text/javascript">
function clear_and_draw(sdate, edate){
$('.plot').empty();
n=data.length
plot_diff = $('input[name=diff_plot]').is(':checked') ? 1 : 0 ;
for(i=0; i < n; i++){
var div_id = "fig"+(i+1);
if (data[i]==null) continue;
if ($("#"+div_id).length==0){
$("body").append('<a href="#'+div_id+'"><div class="plot" id="'+div_id+'"></div></a>');
}
if (data[i].plot_type=="timeseries"){
plots.time_series_plot(div_id,data[i],plot_diff,sdate,edate);
}else if (data[i].plot_type=="exceedance"){
plots.exceedance_plot(div_id,data[i],null,sdate,edate);
}
}
// update links
};
$(document).ready(clear_and_draw(null,null));
$('#system-water-balance-table tr').click(function(){
var_name = $($(this).find('td')[0]).text();
var svg_element = $('div').filter(function(){ return $(this).text().indexOf(var_name)>=0;})
if (svg_element && svg_element.length > 0){
anchor_name = $($(svg_element[0]).parent()).attr("href");
window.location.href=window.location.href.split('#')[0]+anchor_name;
}
});
</script>
<script type="text/javascript">
function extract_date(date_str){
date_fields=date_str.split(",");
return new Date(date_fields[0],date_fields[1],date_fields[2]);
}
$('#time-window-select').change(function(){
var changed_val = $('#time-window-select option:selected').val().split("-");
clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1]));
});
$('#threshold').change(function(){
set_diff_threshold($('#threshold').val());
});
$('input[name=diff_plot]').change(function(){
var changed_val = $('#time-window-select option:selected').val().split("-");
clear_and_draw(extract_date(changed_val[0]),extract_date(changed_val[1]));
});
function set_diff_threshold(threshold){
var diff_marker = function(){
if(Math.abs(parseFloat($(this).text())) >= threshold){
$(this).addClass('large-diff');
}else{
$(this).removeClass('large-diff');
};
};
$('tr td:nth-child(5)').each(diff_marker);
$('tr td:nth-child(9)').each(diff_marker);
$('tr td:nth-child(13)').each(diff_marker);
};
set_diff_threshold($('#threshold').val());
</script>
</body>
</html>"""
fh.close()
logging.debug('Closed out data file')
def compare_dss_files(file1,
file2,
showPlot=False,
outputFile=None,
outputPathFile=None):
"""
Simply compares the files and outputs differences if any of those that differ and lists mismatching pathnames in either
"""
g1 = vutils.opendss(file1)
g2 = vutils.opendss(file2)
print 'Comparing %s to %s' % (file1, file2)
print '%12s\t%32s' % ('DIFFERENCE', 'PATHNAME')
if outputPathFile:
opf_handle = open(outputPathFile, 'wb')
opf = csv.writer(opf_handle,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
opf.writerow([file1, file2])
no_diff = True
for ref1 in g1:
p1 = ref1.pathname
found = False
for ref2 in g2:
p2 = ref2.pathname
if matches(p1, p2):
found = True
diff = ref2.data - ref1.data
absdiff = diff.createSlice(diff.getTimeWindow())
vtimeseries.apply(absdiff, math.fabs)
diff_total = vtimeseries.total(absdiff)
if (diff_total > 1e-06):
no_diff = False
if showPlot: plot(ref1.data, ref2.data)
print '%10.2f\t%32s' % (diff_total, p1)
if outputFile:
diffp = set_part(
p1, 'DIFF-%s-%s' %
(os.path.basename(file1), os.path.basename(file2)),
Pathname.A_PART)
writedss(outputFile, str(diffp), diff)
if outputPathFile:
opf.writerow([p1, p1, diff_total])
break
if (not found):
no_diff = False
print 'No matching path: %s in file %s NOT found in file %s' % (
p1, file1, file2)
if outputPathFile: opf.writerow([p1, "", "M"])
for ref2 in g2:
p2 = ref2.pathname
found = False
for ref1 in g1:
p1 = ref1.pathname
if matches(p1, p2):
found = True
break
if (not found):
no_diff = False
print 'No matching path: %s in file %s NOT found in file %s' % (
p2, file2, file1)
if outputPathFile: opf.writerow(["", p2, "M"])
if no_diff:
print 'NO DIFFERENCE ACROSS ENTIRE FILEs %s and %s' % (file1, file2)
if outputPathFile: opf_handle.close()