def compress(self, ):
'''Close and compress the HDF file after creation.
Notes
-----
This is very important if lots of files have been appended to an
existing HDF file.
'''
# Close the hdf file
self.close()
# Make a copy of the file.
# This compresses the file if a lot of changes have been made
tb.copyFile(self.filename, self.filename + 'temp', overwrite=True)
os.remove(self.filename)
os.rename(self.filename + 'temp', self.filename)
output_array = input1_array
elif ent_name2 in set_ent_names2:
output_array = input2_array
else:
raise Exception("this shouldn't have happened")
table1.append(output_array)
table1.removeRows(start1, stop1)
table1.flush()
print " done."
input1_file.close()
input2_file.close()
if __name__ == '__main__':
import sys
import platform
print "LIAM HDF5 merge %s using Python %s (%s)\n" % \
(__version__, platform.python_version(), platform.architecture()[0])
tables.copyFile("C:/Myliam2/Model/SimulTest.h5", "C:/Myliam2/Model/SimulTestTemp.h5", overwrite=True)
merge_h5("C:/Myliam2/Model/SimulTest.h5", output+"LiamLeg.h5",None)
plt.text(0.5, ints.max()*0.8, text_string.format(slope, intercept, r**2))
plt.savefig(os.path.join(args.cal_folder, name+'_cal_curve'), dpi=200)
plt.close()
if __name__ == '__main__':
h5f, table = cal_h5_build(args)
refs, ref_files = cal_file('calibration.csv')
gcms.clear_png(args.cal_folder)
if args.nproc == 1:
aias = [aia_build(i) for i in ref_files]
else:
p = Pool(args.nproc)
aias = p.map(aia_build, ref_files)
aias = dict( zip(ref_files, aias) )
for name in refs:
int_extract(name, refs[name], aias, args)
h5f.flush()
h5f.close()
pyt.copyFile(args.cal_name, args.cal_name+'temp', overwrite=True)
os.remove(args.cal_name)
os.rename(args.cal_name+'temp', args.cal_name)
def setUp(self):
path = "./test_corpus.h5"
multi_path = "./test_corpus_working.h5"
self.num_W = 26
self.num_D = 100
self.num_Z = 20
# Generate a vocabulary.
vdict = {}
i = 0
for l in string.letters[0:26]:
vdict[i] = l
i += 1
f = t.openFile(path, "w")
g = f.createGroup("/", "g")
v = f.createTable(g, "vocabulary", Word)
for key, value in vdict.iteritems():
w = v.row
w['string'] = value
w['index'] = key
w.append()
v.flush()
f.flush()
# Generate a random corpus with integer word-counts.
dist = [ 0 for i in xrange(20) ] + [ 1, 1, 1, 2, 2, 3 ]
documents = [ np.array([ dist[random.randint(0, 25)] for x in xrange(self.num_W-1) ]+[1]).reshape(1, 26) for i in xrange(self.num_D) ]
maxs = [ np.max(doc) for doc in documents ]
print min(maxs)
dw = f.createEArray("/g", "document_word", atom=tables.Float64Atom(), expectedrows=20, shape=(0, 26))
for d in documents:
dw.append(d)
dw.flush()
f.flush()
# Generate random probability matrices.
d_t = f.createEArray("/g", "document_topic", atom=tables.Float64Atom(), expectedrows=self.num_D, shape=(0, self.num_Z))
for i in xrange(self.num_D):
vec = np.random.random(size=(self.num_Z, 1))
normalize(vec)
d_t.append(vec.reshape(1, self.num_Z))
f.flush()
t_w = f.createEArray("/g", "topic_word", atom=tables.Float64Atom(), expectedrows=self.num_Z, shape=(self.num_Z, 0))
for i in xrange(self.num_W):
vec = np.random.random(size=(self.num_Z, 1))
normalize(vec)
t_w.append(vec)
f.flush()
t_ = f.createEArray("/g", "topic", atom=tables.Float64Atom(), expectedrows=self.num_D, shape=(0, self.num_W, self.num_Z))
for i in xrange(self.num_D):
mat = []
for x in xrange(self.num_W):
vec = np.random.random(size=(self.num_Z, 1))
normalize(vec)
mat.append(vec.reshape(self.num_Z))
a = np.array([mat])
t_.append(a)
f.flush()
f.close()
# Make a copy, for comparison.
t.copyFile(path, multi_path)
