def combine_simulations(simulations,
station_path,
output_file,
copy_tables=DEFAULT_TABLES,
verbose=False,
progress=False):
"""
Combines the result of multiple results from a simulation
:param simulations: list of paths to simulation files
:param station_path: the station path in the h5 file to copy (for now only 1
station is supported)
:param output_file: the output file
:param copy_tables: the tables in the file to copy (relative to the station_path)
:param verbose: print debugging information
:param progress: show progress
:return:
"""
if progress:
print('Creating tables')
tables.copy_file(simulations[0], output_file, overwrite=True)
with tables.open_file(output_file, 'a') as output:
if progress:
iterator = pbar(simulations[1:])
else:
iterator = simulations[1:]
for sim in iterator:
with tables.open_file(sim, 'r') as data:
if station_path not in data:
if verbose:
print('%s not populated' % sim)
continue
for to_copy in copy_tables:
path = tables.path.join_path(station_path, to_copy)
copied_table = data.get_node(path)
to_table = output.get_node(path)
row = to_table.row
length_table = len(to_table)
for event in copied_table:
for key in copied_table.colnames:
if key == 'event_id':
row[key] = length_table + event['event_id']
else:
row[key] = event[key]
row.append()
to_table.flush()
def convertFile(filename):
# Need to handle a both cases where filename includes a _dumpstep.h5 and where it is a plain filename
# if there are multiple matches,
print "Reorder rows and columns attempting to match against: "+filename
if isvsh5suffix(filename):
filenamepieces=filename.split('_')
print len(filenamepieces)
if len(filenamepieces)==4:
(simbase,dataset,rank,dumpnoAndExt)=filenamepieces
elif len(filenamepieces)==3:
(simbase,dataset,dumpnoAndExt)=filenamepieces
# outfilename = filename[:-5]+"C.vsh5"
else:
print "unknown file format - don't know how to tell dumps, rank, dataset from filename"
exit()
tdsetname=dataset+"N"
outfilename = simbase+"_"+dataset+"_C_"+dumpnoAndExt
elif ish5suffix(filename):
filenamepieces=filename.split('_')
print len(filenamepieces)
if len(filenamepieces)==4:
(simbase,dataset,rank,dumpnoAndExt)=filenamepieces
elif len(filenamepieces)==3:
(simbase,dataset,dumpnoAndExt)=filenamepieces
# outfilename = filename[:-5]+"C.vsh5"
else:
print "unknown file format - don't know how to tell dumps, rank, dataset from filename"
exit()
tdsetname=dataset+"N"
outfilename = simbase+"_"+dataset+"_C_"+dumpnoAndExt
else:
print"need to figure out if we're here, as there will be multiple output files"
print outfilename
tables.copy_file(filename,outfilename,overwrite=1)
h5=tables.open_file(outfilename,'r+')
print h5.root.aperp.shape
# While this works it may be better to just read using numpy.asfortranarray()
# ie fieldin=numpy.asfortranarray(h5.root.aperp.read()) or whatever actually works
fieldin=h5.root.aperp.read()
print "The input has "+str(len(fieldin.shape))+" dimensions"
print "fieldin shape "+str(fieldin.shape)
testfield=numpy.asarray(fieldin, order='C')
print "testfield shape"+str(testfield.shape)
testfield=numpy.asfortranarray(fieldin)
print "testfield shape"+str(testfield.shape)
if len(fieldin.shape)==4:
fieldout=numpy.zeros((fieldin.shape[3],fieldin.shape[2],fieldin.shape[1],fieldin.shape[0]))
print "fieldout shape"+str(fieldout.shape)
print "3d plus component"
for i in range(fieldin.shape[0]):
for j in range(fieldin.shape[2]):
fieldout[:,j,:,i]=fieldin[i,:,j,:].T
elif len(fieldin.shape)==2:
fieldout=fieldin.T
else:
print "Do not recognise structure of file "+filename+" setting the _C file as just a copy"
print fieldout.shape
h5.create_array('/','aperpN',fieldout)
h5.copy_node_attrs('/aperp','/aperpN')
h5.remove_node('/aperp')
h5.rename_node('/aperpN','aperp')
h5.root._v_children['aperp']._v_attrs.vsIndexOrder="compMinorC"
h5.root._v_children['meshScaled']._v_attrs.vsIndexOrder="compMinorC"
h5.root._v_children['meshScaled']._v_attrs.vsAxisLabels="x,y,z2"
def swapXYZ(tb):
bx = tb[2]
by = tb[1]
bz2 = tb[0]
tbn = [bx,by,bz2]
return tbn
if len(fieldin.shape)==4:
tlb = h5.root._v_children['meshScaled']._v_attrs.vsLowerBounds
nlb = swapXYZ(tlb)
h5.root._v_children['meshScaled']._v_attrs.vsLowerBounds=numpy.array(nlb)
tub = h5.root._v_children['meshScaled']._v_attrs.vsUpperBounds
nub = swapXYZ(tub)
h5.root._v_children['meshScaled']._v_attrs.vsUpperBounds = numpy.array(nub)
tnc = h5.root._v_children['meshScaled']._v_attrs.vsNumCells
nnc = swapXYZ(tnc)
h5.root._v_children['meshScaled']._v_attrs.vsNumCells = numpy.array(nnc)
h5.root._v_children['globalLimits']._v_attrs.vsLowerBounds = numpy.array(nlb)
h5.root._v_children['globalLimits']._v_attrs.vsUpperBounds = numpy.array(nub)
h5.close()
print("xi: "+str(xi)+" yi: "+str(yi))
magx,phasex,freqx=getMagPhase(h5,xi,yi,0)
magy,phasey,freqy=getMagPhase(h5,xi,yi,1)
# s0=numpy.max(numpy.add(numpy.square(magx),numpy.square(magy)),1.e-99)
# s1=numpy.subtract(numpy.square(magx),numpy.square(magy))
# s2=2*numpy.multiply(numpy.multiply(magx,magy),numpy.cos(numpy.subtract(phasex,phasey)))
# s3=2*numpy.multiply(numpy.multiply(magx,magy),numpy.cos(numpy.subtract(phasex,phasey)))
# various averages are calculated using stokesLength, but they don't appear to be used.
StokesParams[xi,yi,:,0]=magx
StokesParams[xi,yi,:,1]=phasex
StokesParams[xi,yi,:,2]=freqx
StokesParams[xi,yi,:,3]=magy
StokesParams[xi,yi,:,4]=phasey
StokesParams[xi,yi,:,5]=freqy
#writeFieldOutput3D(StokesParams,filename)
tables.copy_file(inputFilename,outFilename,overwrite=1)
h5out=tables.open_file(outFilename,'r+')
dn='stokes' #dataname - shortening
h5out.create_array('/',dn,StokesParams)
h5out.copy_node_attrs('/aperp','/stokes')
h5out.remove_node('/aperp')
h5out.root.stokes._v_attrs.vsLabels="magx,phasex,freqx,magy,phasey,freqy"
h5out.create_group('/','s0','normalization of stokes parameters')
h5out.root.s0._v_attrs.vsType="vsVars"
h5out.root.s0._v_attrs.s0="max((sqr(magx)+sqr(magy)),1.e-99)"
h5out.create_group('/','P1','Stokes parameter P1')
h5out.root.P1._v_attrs.vsType="vsVars"
h5out.root.P1._v_attrs.P1="(sqr(magx)-sqr(magy))/s0"
h5out.create_group('/','P2','Stokes parameter P2')
h5out.root.P2._v_attrs.vsType="vsVars"
h5out.root.P2._v_attrs.P2="2*magx*magy*cos(phasex-phasey)/s0"
f1=tables.open_file(loopfname)
fieldName=dataset+"_"+fieldPos+"_"+component+"_"+str(rank)
fieldNameR=dataset+"_"+fieldPos+"_real_"+str(rank)
if component=='real':
# fieldout[:,:,NZ2start:(NZ2start+thisNZ2[fieldName]+1),0]=f1.root._v_children[dataset+"_"+fieldPos+"_"+component].read().T
fieldout[:,:,NZ2start:(NZ2start+thisNZ2[fieldNameR]),0]=f1.root._v_children[dataset+"_"+fieldPos+"_"+component].read().T
# fieldout[:,:,NZ2start:(NZ2start+thisNZ2[fieldNameR]),0]=f1.root._v_children[dataset+"_"+fieldPos+"_"+component].read().reshape(NX,NY,thisNZ2[fieldNameR])
NZ2start+=thisNZ2[fieldNameR]
if component=='imag':
fieldout[:,:,NZ2start:(NZ2start+thisNZ2[fieldNameR]),1]=f1.root._v_children[dataset+"_"+fieldPos+"_"+component].read().T
# fieldout[:,:,NZ2start:(NZ2start+thisNZ2[fieldNameR]),1]=f1.root._v_children[dataset+"_"+fieldPos+"_"+component].read().reshape(NX,NY,thisNZ2[fieldNameR])
NZ2start+=thisNZ2[fieldNameR]
f1.close()
# except:
# print "could not use "+simbase+"_"+dataset+"_"+fieldPos+"_"+component+"_"+str(rank)+"_"+dumpnoAndExt+", does it exist?"
tables.copy_file(filename,outfilename,overwrite=1)
h5=tables.open_file(outfilename,'r+')
copiedDsetName=dataset+"_"+fieldpart+"_"+dsetcomponent
print h5.root._v_children[copiedDsetName].shape
print copiedDsetName
# While this works it may be better to just read using numpy.asfortranarray()
# ie fieldin=numpy.asfortranarray(h5.root.aperp.read()) or whatever actually works
#a1=f1.root.aperp_front_real.read()
#a2=f2.root.aperp_active_real.read()
#a3=f3.root.aperp_back_real.read()
#fieldout=numpy.zeros((nx,ny,nz,nc))
#fieldin=h5.root.aperp.read()
#print "fieldin shape "+str(fieldin.shape)
#testfield=numpy.asarray(fieldin, order='C')
#print "testfield shape"+str(testfield.shape)
#testfield=numpy.asfortranarray(fieldin)
thisProc=int(procFile.split(os.sep)[-1].split('.')[0].split('_')[-2])
procNos.append(thisProc)
print "maxProcNo: "+str(max(procNos))
if max(procNos) != numProcs:
print "This isn't very good, is it!"
return numProcs, sorted(procNos)
baseName="fig7_electrons"
#baseName="electrons"
filelist, dumpStepNos=getTimeSlices(baseName)
numProcs0, procNos0=getNumProcs(filelist,baseName,0)
for i in range(len(filelist)):
for j in range(numProcs0):
outfilename=baseName+'_'+str(dumpStepNos[i])+'.vsh5'
if j==0:
tables.copy_file(filelist[0], outfilename,overwrite=True)
else:
print "Processor: "+str(j)
h5out=tables.open_file(outfilename,'r+')
h5in=tables.open_file(baseName+'_'+str(j)+'_'+str(dumpStepNos[i])+'.h5','r')
oldElecs=h5out.root.electrons.read()
newElecs=h5in.root.electrons.read()
elecs=numpy.zeros((oldElecs.shape[0]+newElecs.shape[0],newElecs.shape[1]))
elecs[0:oldElecs.shape[0],:]=oldElecs
elecs[oldElecs.shape[0]:,:]=newElecs
h5out.create_array('/','newElecs',elecs)
h5out.copy_node_attrs('/electrons','/newElecs')
h5out.remove_node('/electrons')
h5out.rename_node('/newElecs','electrons')
h5in.close()
h5out.close()
#!/usr/bin/python # -*- coding: utf8 -*- # cp936 import tables as tb dir = "c:/stock/" tb.copy_file(dir + "sh_day.h5", dir + "sh_day_new.h5") tb.copy_file(dir + "sz_day.h5", dir + "sz_day_new.h5") tb.copy_file(dir + "sh_5min.h5", dir + "sh_5min_new.h5") tb.copy_file(dir + "sz_5min.h5", dir + "sz_5min_new.h5") #tb.copy_file(dir + "sh_1min.h5", dir + "sh_1min_new.h5") #tb.copy_file(dir + "sz_1min.h5", dir + "sz_1min_new.h5")