def test_Track_read(track):
# test Track data format
from etrack.io import trackio
import os
filebase = ''.join(
chr(i) for i in np.random.randint(97, 122, size=(8, )))
filename = '.'.join([filebase, 'h5'])
with h5py.File(filename, 'a') as h5file:
trackio.write_object_to_hdf5(track, h5file, 'track')
with h5py.File(filename, 'r') as h5file:
track2 = trackio.read_object_from_hdf5(h5file['track'])
assert track2['is_modeled'] == track.is_modeled
assert track2['pixel_size_um'] == track.pixel_size_um
assert track2['noise_ev'] == track.noise_ev
assert track2['label'] == track.label
assert track2['energy_kev'] == track.energy_kev
assert np.all(track2['image'] == track.image)
assert track2['algorithms']['python HT v1.5']['alpha_deg'] == 120.5
assert track2['algorithms']['python HT v1.5']['beta_deg'] == 43.5
test_Track_from_pydict(track, track2)
test_Track_from_hdf5()
os.remove(filename)
def pyobjs_to_h5(g4tracks, pixnoise, filename, v):
"""
Given the python object lists, write to HDF5 file at filename.
"""
with h5py.File(filename, 'w', libver='latest', driver='core') as f:
for ind in range(len(g4tracks)):
pyobj_to_h5 = {}
indstr = '{:05d}'.format(ind)
g = f.create_group(indstr)
# write g4track
if g4tracks[ind]:
filejob.vprint(
v, 3,
'Writing ' + str(ind) + ' g4track at ' + time.ctime())
trackio.write_object_to_hdf5(g4tracks[ind],
g,
'g4track',
pyobj_to_h5=pyobj_to_h5)
# write each pixnoise
for key, val in pixnoise.iteritems():
filejob.vprint(
v, 4,
'Writing ' + str(ind) + ' ' + key + ' at ' + time.ctime())
if isinstance(val[ind], trackdata.Track):
trackio.write_object_to_hdf5(val[ind],
g,
key,
pyobj_to_h5=pyobj_to_h5)
else:
# track error code
t = g.create_group(key)
t.attrs['errorcode'] = val[ind]
def test_Track_read(track):
# test Track data format
from etrack.io import trackio
import os
filebase = ''.join(
chr(i) for i in np.random.randint(97, 122, size=(8,)))
filename = '.'.join([filebase, 'h5'])
with h5py.File(filename, 'a') as h5file:
trackio.write_object_to_hdf5(
track, h5file, 'track')
with h5py.File(filename, 'r') as h5file:
track2 = trackio.read_object_from_hdf5(
h5file['track'])
assert track2['is_modeled'] == track.is_modeled
assert track2['pixel_size_um'] == track.pixel_size_um
assert track2['noise_ev'] == track.noise_ev
assert track2['label'] == track.label
assert track2['energy_kev'] == track.energy_kev
assert np.all(track2['image'] == track.image)
assert track2['algorithms']['python HT v1.5']['alpha_deg'] == 120.5
assert track2['algorithms']['python HT v1.5']['beta_deg'] == 43.5
test_Track_from_pydict(track, track2)
test_Track_from_hdf5()
os.remove(filename)
def pyobjs_to_h5(g4tracks, pixnoise, filename, v):
"""
Given the python object lists, write to HDF5 file at filename.
"""
with h5py.File(filename, "w", libver="latest", driver="core") as f:
for ind in range(len(g4tracks)):
pyobj_to_h5 = {}
indstr = "{:05d}".format(ind)
g = f.create_group(indstr)
# write g4track
if g4tracks[ind]:
filejob.vprint(v, 3, "Writing " + str(ind) + " g4track at " + time.ctime())
trackio.write_object_to_hdf5(g4tracks[ind], g, "g4track", pyobj_to_h5=pyobj_to_h5)
# write each pixnoise
for key, val in pixnoise.iteritems():
filejob.vprint(v, 4, "Writing " + str(ind) + " " + key + " at " + time.ctime())
if isinstance(val[ind], trackdata.Track):
trackio.write_object_to_hdf5(val[ind], g, key, pyobj_to_h5=pyobj_to_h5)
else:
# track error code
t = g.create_group(key)
t.attrs["errorcode"] = val[ind]
def classify_etc(loadfile, savefile, v):
"""
1. Moments algorithm
2. HybridTrack algorithm
3. Classify
"""
progressflag = not file_vars()[0]
pn = 'pix10_5noise15'
HTname = 'python HT v1.52'
MTname = 'moments v1.0'
vprint(v, 1, 'Starting {} at {} with {}% mem usage'.format(
loadfile, time.ctime(), psutil.virtual_memory().percent))
pyobj_to_h5 = {}
try:
with h5py.File(loadfile, 'a', driver='core') as f, h5py.File(
savefile, 'a', driver='core') as h5save:
#
n = 0
if progressflag:
pbar = progressbar.ProgressBar(
widgets=[progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.ETA()], maxval=len(f))
pbar.start()
keylist = f.keys()
keylist.sort()
for ind in keylist:
vprint(v, 3, 'Beginning track {} in {}'.format(ind, loadfile))
if int(ind) % 50 == 0:
vprint(v, 2,
'Beginning track {} in {}'.format(ind, loadfile))
# evt is for the G4Track object
evt = f[ind]
# shouldn't get a KeyError because we are looping over ind
# trk is for the Track object
try:
trk = evt[pn]
except KeyError:
vprint(v, 1,
'**Missing key {} in {}{}, skipping'.format(
pn, loadfile, evt.name))
continue
trkpath = ind + '/' + pn
errorcode = 0
n += 1
if n > 50:
# testing
# vprint(v, 1, 'Finished 50 files, exiting')
# break
pass
# load G4Track
vprint(v, 3, 'Loading G4Track {} in {}'.format(ind, loadfile))
try:
this_g4track = G4Track.from_dth5(evt)
except trackio.InterfaceError:
vprint(v, 2, 'InterfaceError in G4Track at {}, {}'.format(
loadfile, ind))
continue
# load Track
vprint(v, 3, 'Loading Track {} in {}'.format(ind, loadfile))
try:
this_track = Track.from_dth5(trk, g4track=this_g4track)
except trackio.InterfaceError:
vprint(v, 2, 'InterfaceError in Track at {}, {}'.format(
loadfile, ind))
continue
# check for error codes - Track.from_dth5 may return an int
if isinstance(this_track, int):
this_save = h5save.create_group(trkpath)
this_save.attrs.create(
'errorcode', this_track, shape=np.shape(this_track))
continue
# run moments algorithm
vprint(v, 3, 'Running moments on track {} in {}'.format(
ind, loadfile))
try:
mom = tm.MomentsReconstruction(this_track.image)
mom.reconstruct()
except ht.NoEndsFound:
mom.alpha = None
errorcode = 4
this_save = h5save.create_group('mom_' + trkpath)
this_save.attrs.create(
'errorcode', errorcode, shape=np.shape(errorcode))
else:
# write into savefile
trackio.write_object_to_hdf5(
mom, h5save, 'mom_' + trkpath, pyobj_to_h5=pyobj_to_h5)
# write into track object
if mom.alpha:
this_track.add_algorithm(
MTname,
alpha_deg=mom.alpha * 180 / np.pi,
beta_deg=np.nan, info=None)
else:
this_track.add_algorithm(
MTname,
alpha_deg=np.nan,
beta_deg=np.nan, info=None)
# run HT algorithm (v1.52)
vprint(v, 3, 'Running HT on track {} in {}'.format(
ind, loadfile))
try:
_, HTinfo = ht.reconstruct(this_track)
except (ht.InfiniteLoop, ht.NoEndsFound):
# write empty into track object
this_track.add_algorithm(
HTname,
alpha_deg=np.nan,
beta_deg=np.nan,
info=None)
else:
# trim memory usage
if hasattr(HTinfo, 'ridge'):
if HTinfo.ridge:
for ridgept in HTinfo.ridge:
ridgept.cuts = None
ridgept.best_cut = None
# write into track object
this_track.add_algorithm(
HTname,
alpha_deg=HTinfo.alpha_deg,
beta_deg=HTinfo.beta_deg,
info=HTinfo)
# write py HDF5 format into savefile (including alg outputs)
try:
trackio.write_object_to_hdf5(
this_track, h5save, trkpath,
pyobj_to_h5=pyobj_to_h5)
except trackio.InterfaceError:
vprint(v, 1, 'InterfaceError writing to file at ' +
'{}, {}'.format(savefile, ind))
continue
# run classifier
vprint(v, 3, 'Running MC classifier on track {} in {}'.format(
ind, loadfile))
classifier = cl.Classifier(this_track.g4track)
try:
classifier.mc_classify()
except cl.TrackTooShortError:
classifier.error = 'TrackTooShortError'
else:
vprint(v, 3,
'Running ends classifier on track {} in {}'.format(
ind, loadfile))
try:
classifier.end_classify(this_track, mom=mom)
except tp.G4TrackTooBigError:
classifier.error = 'G4TrackTooBigError'
except cl.NoEnds:
classifier.error = 'NoEnds'
# write into savefile
vprint(v, 3, 'Writing classifier into {} for track {}'.format(
savefile, ind))
trackio.write_object_to_hdf5(
classifier, h5save, 'cl_' + trkpath,
pyobj_to_h5=pyobj_to_h5)
if progressflag:
pbar.update(n)
if progressflag:
pbar.finish()
# f gets closed here
except NotImplementedError:
pass
def run_main():
serverflag = True
doneflag = False # only take files that have an associated done file
if serverflag:
loadpath = '/global/home/users/bcplimley/multi_angle/HTbatch01_AR151/'
savepath = loadpath
else:
loadpath = '/media/plimley/TEAM 7B/HTbatch01_AR/'
savepath = loadpath
loadglob = 'MultiAngle_HT_*_*_AR.h5'
savename = 'compile_AR_' + str(int(time.time())) + '.h5'
doneglob = 'done_MultiAngle_HT_*_*_AR.h5'
if doneflag:
flist = glob.glob(os.path.join(loadpath, doneglob))
done2load = get_filename_function(doneglob, loadglob)
else:
flist = glob.glob(os.path.join(loadpath, loadglob))
print('flist contains {} files'.format(str(len(flist))))
pnlist = [
'pix2_5noise0',
'pix5noise0',
'pix5noise15',
'pix10_5noise0',
'pix10_5noise15',
'pix10_5noise20',
'pix10_5noise50',
'pix10_5noise100',
'pix10_5noise200',
'pix10_5noise500',
'pix10_5noise1000',
'pix10_5noise2000',
'pix20noise0',
'pix40noise0',
]
alglist = [
'python HT v1.52',
'python HT v1.52b',
'python HT v1.52c',
'python HT v1.52d',
]
AR = {}
for fname in flist:
if doneflag:
donename = os.path.split(fname)[-1]
loadname = done2load(donename)
loadfile = os.path.join(loadpath, loadname)
else:
loadfile = fname
loadname = fname
try:
with h5py.File(loadfile, 'r') as h5f:
for pn in pnlist:
if pn not in AR:
AR[pn] = {}
for alg in alglist:
try:
this_AR = evaluation.AlgorithmResults.from_hdf5(
h5f[pn][alg])
except KeyError:
# happens once somewhere
print('KeyError on {} - {} - {}'.format(
loadname, pn, alg))
continue
try:
AR[pn][alg] += this_AR
except KeyError:
AR[pn][alg] = this_AR
except IOError:
print('IOError on {}; skipping'.format(loadfile))
print('saving to {}'.format(os.path.join(savepath, savename)))
with h5py.File(os.path.join(savepath, savename), 'w') as fsave:
for pn in pnlist:
fsave.create_group(pn)
for alg in alglist:
AR[pn][alg].add_default_uncertainties()
trackio.write_object_to_hdf5(AR[pn][alg], fsave[pn], alg)
def pyml_run_algs(loadfile, savefile, v):
progressflag = False # turn off for parallel
pnlist = [
'pix2_5noise0',
'pix5noise0',
'pix5noise15',
'pix10_5noise0',
'pix10_5noise15',
'pix10_5noise20',
'pix10_5noise50',
'pix10_5noise100',
'pix10_5noise200',
'pix10_5noise500',
'pix10_5noise1000',
'pix10_5noise2000',
'pix20noise0',
'pix40noise0',
]
alglist = {
'python HT v1.52': ht,
# 'python HT v1.52a': ht_a,
# 1.52a causes RuntimeWarning and possibly breaks something more...
'python HT v1.52b': ht_b,
'python HT v1.52c': ht_c,
'python HT v1.52d': ht_d,
}
tracklist = {}
AR = {}
for pnname in pnlist:
tracklist[pnname] = []
AR[pnname] = {}
for algname in alglist.keys():
AR[pnname][algname] = []
vprint(v, 1, 'Starting {} at {} with {}% mem usage'.format(
loadfile, time.ctime(), psutil.virtual_memory().percent))
try:
with h5py.File(loadfile, 'a', driver='core') as h5load:
filename = h5load.filename
n = 0
if progressflag:
pbar = progressbar.ProgressBar(
widgets=[progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.ETA()], maxval=len(h5load))
pbar.start()
keylist = h5load.keys()
keylist.sort()
for ind in keylist:
if int(ind) % 10 == 0:
vprint(v, 2, ' Running {} #{} at {}'.format(
loadfile, ind, time.ctime()))
trk = h5load[ind]
h5_to_pydict = {}
pydict_to_pyobj = {}
pyobj_to_h5 = {}
n += 1
if n > 50:
# pdb.set_trace()
# continue # TODO temp!
pass
for pnname in pnlist:
try:
pn = trk[pnname]
except KeyError:
vprint(v, 1,
'**Missing key {} in {}{}, skipping'.format(
pnname, loadfile, trk.name))
continue
vprint(v, 3, 'Running {}{} at {}'.format(
loadfile, pn.name, time.ctime()))
# load track
try:
this_track = trackdata.Track.from_hdf5(
pn,
h5_to_pydict=h5_to_pydict,
pydict_to_pyobj=pydict_to_pyobj)
except trackio.InterfaceError:
vprint(v, 2, 'InterfaceError at {}{}'.format(
loadfile, pn.name))
continue
tracklist[pnname].append(this_track)
# each algorithm version
for algname, algfunc in alglist.items():
vprint(v, 4, 'Running alg {} at {}'.format(
algname, time.ctime()))
# check for result
if algname not in this_track.algorithms:
# run algorithm
try:
_, HTinfo = algfunc.reconstruct(this_track)
except algfunc.InfiniteLoop:
continue
except algfunc.NoEndsFound:
continue
# trim memory usage!
if hasattr(HTinfo, 'ridge'):
if HTinfo.ridge:
for ridgept in HTinfo.ridge:
ridgept.cuts = None
ridgept.best_cut = None
# write into track object
try:
this_track.add_algorithm(
algname,
alpha_deg=HTinfo.alpha_deg,
beta_deg=HTinfo.beta_deg,
info=HTinfo)
# write into HDF5
trackio.write_object_to_hdf5(
this_track.algorithms[algname],
pn['algorithms'], algname,
pyobj_to_h5=pyobj_to_h5)
except trackdata.InputError:
# already has this algorithm
pass
if progressflag:
pbar.update(n)
if progressflag:
pbar.finish()
# h5load gets closed
vprint(
v, 1.5,
'\n Finished loading {} at {} with {}% mem usage'.format(
loadfile, time.ctime(), psutil.virtual_memory().percent))
except IOError:
vprint(v, 1, 'IOError: Unable to open file (I think) for {}'.format(
loadfile))
return None
# AlgorithmResults objects
alglist2 = alglist.keys() # + ['matlab HT v1.5']
for pnname in pnlist:
for algname in alglist2:
this_AR = evaluation.AlgorithmResults.from_track_list(
tracklist[pnname], alg_name=algname, filename=filename)
AR[pnname][algname] = this_AR
vprint(v, 2, '\n Created AR objects for {} at {}'.format(
loadfile, time.ctime()))
# write to savefile
try:
with h5py.File(savefile, 'w', driver='core') as h5save:
for pnname, AR_pn in AR.items():
pngroup = h5save.create_group(pnname)
for algname, this_AR in AR_pn.items():
trackio.write_object_to_hdf5(this_AR, pngroup, algname)
vprint(v, 1.5,
'Finished saving {} at {}'.format(savefile, time.ctime()))
except IOError:
vprint(v, 1, 'IOError: Unable to create file (I think) for {}'.format(
savefile))
return None
def pyml_run_algs(loadfile, savefile, v):
progressflag = False # turn off for parallel
pnlist = [
'pix2_5noise0',
'pix5noise0',
'pix5noise15',
'pix10_5noise0',
'pix10_5noise15',
'pix10_5noise20',
'pix10_5noise50',
'pix10_5noise100',
'pix10_5noise200',
'pix10_5noise500',
'pix10_5noise1000',
'pix10_5noise2000',
'pix20noise0',
'pix40noise0',
]
alglist = {
'python HT v1.52': ht,
# 'python HT v1.52a': ht_a,
# 1.52a causes RuntimeWarning and possibly breaks something more...
'python HT v1.52b': ht_b,
'python HT v1.52c': ht_c,
'python HT v1.52d': ht_d,
}
tracklist = {}
AR = {}
for pnname in pnlist:
tracklist[pnname] = []
AR[pnname] = {}
for algname in alglist.keys():
AR[pnname][algname] = []
vprint(
v, 1, 'Starting {} at {} with {}% mem usage'.format(
loadfile, time.ctime(),
psutil.virtual_memory().percent))
try:
with h5py.File(loadfile, 'a', driver='core') as h5load:
filename = h5load.filename
n = 0
if progressflag:
pbar = progressbar.ProgressBar(widgets=[
progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.ETA()
],
maxval=len(h5load))
pbar.start()
keylist = h5load.keys()
keylist.sort()
for ind in keylist:
if int(ind) % 10 == 0:
vprint(
v, 2, ' Running {} #{} at {}'.format(
loadfile, ind, time.ctime()))
trk = h5load[ind]
h5_to_pydict = {}
pydict_to_pyobj = {}
pyobj_to_h5 = {}
n += 1
if n > 50:
# pdb.set_trace()
# continue # TODO temp!
pass
for pnname in pnlist:
try:
pn = trk[pnname]
except KeyError:
vprint(
v, 1, '**Missing key {} in {}{}, skipping'.format(
pnname, loadfile, trk.name))
continue
vprint(
v, 3,
'Running {}{} at {}'.format(loadfile, pn.name,
time.ctime()))
# load track
try:
this_track = trackdata.Track.from_hdf5(
pn,
h5_to_pydict=h5_to_pydict,
pydict_to_pyobj=pydict_to_pyobj)
except trackio.InterfaceError:
vprint(
v, 2,
'InterfaceError at {}{}'.format(loadfile, pn.name))
continue
tracklist[pnname].append(this_track)
# each algorithm version
for algname, algfunc in alglist.items():
vprint(
v, 4, 'Running alg {} at {}'.format(
algname, time.ctime()))
# check for result
if algname not in this_track.algorithms:
# run algorithm
try:
_, HTinfo = algfunc.reconstruct(this_track)
except algfunc.InfiniteLoop:
continue
except algfunc.NoEndsFound:
continue
# trim memory usage!
if hasattr(HTinfo, 'ridge'):
if HTinfo.ridge:
for ridgept in HTinfo.ridge:
ridgept.cuts = None
ridgept.best_cut = None
# write into track object
try:
this_track.add_algorithm(
algname,
alpha_deg=HTinfo.alpha_deg,
beta_deg=HTinfo.beta_deg,
info=HTinfo)
# write into HDF5
trackio.write_object_to_hdf5(
this_track.algorithms[algname],
pn['algorithms'],
algname,
pyobj_to_h5=pyobj_to_h5)
except trackdata.InputError:
# already has this algorithm
pass
if progressflag:
pbar.update(n)
if progressflag:
pbar.finish()
# h5load gets closed
vprint(
v, 1.5,
'\n Finished loading {} at {} with {}% mem usage'.format(
loadfile, time.ctime(),
psutil.virtual_memory().percent))
except IOError:
vprint(
v, 1,
'IOError: Unable to open file (I think) for {}'.format(loadfile))
return None
# AlgorithmResults objects
alglist2 = alglist.keys() # + ['matlab HT v1.5']
for pnname in pnlist:
for algname in alglist2:
this_AR = evaluation.AlgorithmResults.from_track_list(
tracklist[pnname], alg_name=algname, filename=filename)
AR[pnname][algname] = this_AR
vprint(
v, 2,
'\n Created AR objects for {} at {}'.format(loadfile, time.ctime()))
# write to savefile
try:
with h5py.File(savefile, 'w', driver='core') as h5save:
for pnname, AR_pn in AR.items():
pngroup = h5save.create_group(pnname)
for algname, this_AR in AR_pn.items():
trackio.write_object_to_hdf5(this_AR, pngroup, algname)
vprint(v, 1.5,
'Finished saving {} at {}'.format(savefile, time.ctime()))
except IOError:
vprint(
v, 1,
'IOError: Unable to create file (I think) for {}'.format(savefile))
return None
def run_main():
serverflag = True
doneflag = False # only take files that have an associated done file
if serverflag:
loadpath = '/global/home/users/bcplimley/multi_angle/HTbatch01_AR151/'
savepath = loadpath
else:
loadpath = '/media/plimley/TEAM 7B/HTbatch01_AR/'
savepath = loadpath
loadglob = 'MultiAngle_HT_*_*_AR.h5'
savename = 'compile_AR_' + str(int(time.time())) + '.h5'
doneglob = 'done_MultiAngle_HT_*_*_AR.h5'
if doneflag:
flist = glob.glob(os.path.join(loadpath, doneglob))
done2load = get_filename_function(doneglob, loadglob)
else:
flist = glob.glob(os.path.join(loadpath, loadglob))
print('flist contains {} files'.format(str(len(flist))))
pnlist = [
'pix2_5noise0',
'pix5noise0',
'pix5noise15',
'pix10_5noise0',
'pix10_5noise15',
'pix10_5noise20',
'pix10_5noise50',
'pix10_5noise100',
'pix10_5noise200',
'pix10_5noise500',
'pix10_5noise1000',
'pix10_5noise2000',
'pix20noise0',
'pix40noise0',
]
alglist = [
'python HT v1.52',
'python HT v1.52b',
'python HT v1.52c',
'python HT v1.52d',
]
AR = {}
for fname in flist:
if doneflag:
donename = os.path.split(fname)[-1]
loadname = done2load(donename)
loadfile = os.path.join(loadpath, loadname)
else:
loadfile = fname
loadname = fname
try:
with h5py.File(loadfile, 'r') as h5f:
for pn in pnlist:
if pn not in AR:
AR[pn] = {}
for alg in alglist:
try:
this_AR = evaluation.AlgorithmResults.from_hdf5(
h5f[pn][alg])
except KeyError:
# happens once somewhere
print('KeyError on {} - {} - {}'.format(
loadname, pn, alg))
continue
try:
AR[pn][alg] += this_AR
except KeyError:
AR[pn][alg] = this_AR
except IOError:
print('IOError on {}; skipping'.format(loadfile))
print('saving to {}'.format(os.path.join(savepath, savename)))
with h5py.File(os.path.join(savepath, savename), 'w') as fsave:
for pn in pnlist:
fsave.create_group(pn)
for alg in alglist:
AR[pn][alg].add_default_uncertainties()
trackio.write_object_to_hdf5(AR[pn][alg], fsave[pn], alg)