def run(self):
self.computeDark()
self.Start = self.ShutterStatus.index('1')
myindexes = [i for i in range(self.Start,len(self.taglist)-1) if (i+rank-1)%(size-1) == 0]
print("Rank %i: N tags = %i"%(rank, len(myindexes)))
for i,idx in enumerate(myindexes):
self.Ntotal += 1
#Here get img
self.retrieveData(self.taglist[idx])
if IsHit(self.data - self.dark, self.args['threshold'], self.args['npixels']):
self.Nhits += 1
self.SaveHits.saveHit(self.data - self.dark, self.PhotonEnergy, self.PumpLaserStatus[i+self.Start])
# Save hit (photonenergy of a given index
# h5 - timeresolved
if self.Ntotal % 10 == 0:
md = mpidata()
md.addarray(str(self.taglist[idx]), self.data, self.Ntotal, self.Nhits, self.Nerr,
('', ''))
md.send()
self.Nerr = 0
self.Ntotal = 0
self.Nhits = 0
#self.hitsIDX = ""
#self.nohitsIDX = ""
print("Rank %i finished" %rank)
md.endrun()
def run(self):
#Object used to send info to the master
for idx in self.indexes:
self.Ntotal += 1
#Opening image with fabio
img = self.fabio.open(self.filenames[idx])
#Performing Data Correction (i.e Dark Subtraction and Azimuthal Integration - The mask is used during the Background subtraction)
data = self.DataCorrection.DarkCorrection(img.data[self.roi.xmin:self.roi.xmax, self.roi.ymin:self.roi.ymax], self.args.roi.tuple)
md = mpidata()
#The hit is performed on the corrected image - using the provided mask
if IsHit(self.DataCorrection.CorrectMask(data), self.options['threshold'], self.options['npixels']):
self.Nhits += 1
if self.roi.active:
#If a ROI was used for hit detection, getting the full-size and correct hit (dark only here)
data = self.DataCorrection.DarkCorrection(img.data, (0,self.detector.shape[0],0,self.detector.shape[1]))
#MaxProj = np.maximum(MaxProj, data)
# TODO: Perform Bragg peak localization
self.SaveHits.saveHit(data, self.args.fns[idx])
md.addarray(self.args.fns[idx], data)
#Sending to the master
if self.Nhits % 2 == 0:
md.send()
self.SaveHits.ClosingOpenH5()
md.endrun()
def run(self):
self.path, self.ovl, self.ty = self.geth5path(
self.filenames[self.indexes[0]])
self.detector.overload = self.ovl
self.hitsIDX = ""
self.nohitsIDX = ""
for idx in self.indexes:
#Opening h5 files
self.h5 = h5py.File(self.filenames[idx])
N, shape0, shape1 = self.h5[self.path].shape
for i in self.getImagesIndexes(N, idx):
self.Ntotal += 1
#Performing Data Correction (i.e Dark Subtraction and Azimuthal Integration - The mask is used during the Background subtraction)
data = self.DataCorrection.DarkCorrection(
self.h5[self.path][i, self.roi.xmin:self.roi.xmax,
self.roi.ymin:self.roi.ymax])
#The hit is performed on the corrected image - using the provided mask
#print self.options['threshold'], self.options['npixels']
if IsHit(
self.DataCorrection.CorrectMask(data,
roi=self.roi.active),
self.options['threshold'], self.options['npixels']):
self.Nhits += 1
self.hitsIDX += "%s //%i\n" % (self.filenames[idx], i)
if self.options['roi'].lower() != 'none' and len(
self.options['output_formats']) > 0:
#If a ROI was used for hit detection, getting the full-size and correct hit (dark only here)
data = self.DataCorrection.DarkCorrection(
self.h5[self.path][i, ::])
#MaxProj = np.maximum(MaxProj, data)
# TODO: Perform Bragg peak localization
self.SaveHits.saveHit(
data,
'%s_%i' % (self.filenames[idx].split('.h5')[0], i))
else:
self.nohitsIDX += "%s //%i\n" % (self.filenames[idx], i)
#Sending to the master
if self.Ntotal % 100 == 0:
md = mpidata()
md.addarray(self.filenames[idx], data, self.Ntotal,
self.Nhits, self.Nerr,
(self.hitsIDX, self.nohitsIDX))
md.send()
self.Nerr = 0
self.Ntotal = 0
self.Nhits = 0
self.hitsIDX = ""
self.nohitsIDX = ""
md.endrun()
def runmaster(nClients, args):
t=time.time()
detector = InitDetector(args)
max_proj = np.zeros(detector.shape)
Ntotal = 0
Nhits = 0
Nerr = 0
txt = ""
OutputFileName = os.path.join(args['output_directory'], 'NPC_run%s' % args['num'].zfill(3),
'%s_%s' % (args['filename_root'], args['num'].zfill(3)))
outTxtHit = open('%s.txt' % OutputFileName, 'w')
outTxtRej = open('%s_rejected.txt' % OutputFileName, 'w')
while nClients > 0:
md = mpidata()
md.recv()
for arrinfo in md.small.arrayinfolist:
if arrinfo.name != 'maxproj':
try:
data = getattr(md, arrinfo.name[0])
#print "Data shape:", data.shape
Ntotal += arrinfo.Ntotal
Nhits += arrinfo.Nhits
Nerr += arrinfo.Nerr
txthits, txtnohits = arrinfo.hitsIDX
if len(txthits) > 0:
print >> outTxtHit, txthits
if len(txtnohits) > 0:
print >> outTxtRej, txtnohits
print("Processed: %5i - Hits %5i - Errors : %i"%(Ntotal, Nhits, Nerr))#md.small.Ntotal, md.small.Nhits, md
except AttributeError:
print('The data from file %s have not been sent yet to the master'%arrinfo.name[0])
if md.small.endrun:
nClients -= 1
if nClients == 0:
print("Master is done")
tf = time.time()
print('Needed %4.2f s to lookup at %i events with %i workers'%(tf-t, Ntotal, size-1))
outTxtRej.close()
outTxtHit.close()
def run(self):
self.path , self.ovl, self.ty = self.geth5path(self.filenames[self.indexes[0]])
self.detector.overload = self.ovl
self.hitsIDX = ""
self.nohitsIDX = ""
for idx in self.indexes:
#Opening h5 files
self.h5 = h5py.File(self.filenames[idx])
N, shape0, shape1 = self.h5[self.path].shape
for i in self.getImagesIndexes(N, idx):
self.Ntotal += 1
#Performing Data Correction (i.e Dark Subtraction and Azimuthal Integration - The mask is used during the Background subtraction)
data = self.DataCorrection.DarkCorrection(self.h5[self.path][i,self.roi.xmin:self.roi.xmax, self.roi.ymin:self.roi.ymax])
#The hit is performed on the corrected image - using the provided mask
#print self.options['threshold'], self.options['npixels']
if IsHit(self.DataCorrection.CorrectMask(data, roi = self.roi.active), self.options['threshold'], self.options['npixels']):
self.Nhits += 1
self.hitsIDX +="%s //%i\n" %(self.filenames[idx], i)
if self.options['roi'].lower() != 'none' and len(self.options['output_formats']) > 0:
#If a ROI was used for hit detection, getting the full-size and correct hit (dark only here)
data = self.DataCorrection.DarkCorrection(self.h5[self.path][i,::])
#MaxProj = np.maximum(MaxProj, data)
# TODO: Perform Bragg peak localization
self.SaveHits.saveHit(data, '%s_%i' %(self.filenames[idx].split('.h5')[0], i))
else:
self.nohitsIDX += "%s //%i\n" %(self.filenames[idx], i)
#Sending to the master
if self.Ntotal % 100 == 0:
md = mpidata()
md.addarray(self.filenames[idx], data, self.Ntotal, self.Nhits, self.Nerr,(self.hitsIDX, self.nohitsIDX))
md.send()
self.Nerr = 0
self.Ntotal = 0
self.Nhits = 0
self.hitsIDX = ""
self.nohitsIDX = ""
md.endrun()
def runmaster(nClients, args):
t = time.time()
#detector = InitDetector(args)
#max_proj = np.zeros(detector.shape)
Ntotal = 0
Nhits = 0
Nerr = 0
txt = ""
#OutputFileName = os.path.join(args['output_directory'], 'NPC_run%s' % args['num'].zfill(3),
# 'hits_%s' % (args['num'].zfill(3)))
#outTxtHit = open('%s.txt' % OutputFileName, 'w')
#outTxtRej = open('%s_rejected.txt' % OutputFileName, 'w')
md = mpidata()
while nClients > 0:
md.recv()
for arrinfo in md.small.arrayinfolist:
if arrinfo.name != 'maxproj':
try:
data = getattr(md, arrinfo.name[0])
#print "Data shape:", data.shape
Ntotal += arrinfo.Ntotal
Nhits += arrinfo.Nhits
Nerr += arrinfo.Nerr
#txthits, txtnohits = arrinfo.hitsIDX
#if len(txthits) > 0:
# outTxtHit.write(txthits)
#if len(txtnohits) > 0:
# outTxtHit.write(txthits)print >> outTxtRej, txtnohits
print("Processed: %5i - Hits %5i - Errors : %i" %
(Ntotal, Nhits,
Nerr)) #md.small.Ntotal, md.small.Nhits, md
except AttributeError:
print(
'The data from file %s have not been sent yet to the master'
% arrinfo.name[0])
if md.small.endrun:
nClients -= 1
if nClients == 0:
print("Master is done")
tf = time.time()
print('Needed %4.2f s to lookup at %i events with %i workers' %
(tf - t, Ntotal, size - 1))
if args['crystfel']:
#Should we loop over ranks ??
if args['TimeResolved']:
for laser in ['on', 'off']:
#OutputFilename = os.path.join(self.args['output_directory'],
# 'NPC_run%s' % self.args['num'].zfill(3),
cryst_dir = os.path.join(args['output_directory'],
'NPC_run%s' % args['num'].zfill(3),
'CRYSTFEL_%s' % laser)
#cryst_dir = os.path.join(IO.procdir, IO.H5Dir, 'CRYSTFEL_%s' % laser)
os.mkdir(cryst_dir)
os.chmod(cryst_dir, 777)
# Copy input files to this dir
root_dir = '/home/coquelle/CRYSTFEL_FILES/'
files = [
'current.geom', 'current.pdb', 'index.csh', 'stats.csh',
'flauncher.csh'
]
for f in files:
_f = os.path.join(root_dir, f)
shutil.copy(_f, cryst_dir)
os.chdir(cryst_dir)
h5path = os.path.join(args['output_directory'],
'NPC_run%s' % args['num'].zfill(3),
'HDF5')
subprocess.call("csh flauncher.csh %s %s *%s.h5" %
(h5path, "%s_%s" %
(args['run'], laser), laser),
shell=True)
else:
#cryst_dir = os.path.join(IO.procdir, IO.H5Dir, 'CRYSTFEL')
cryst_dir = os.path.join(args['output_directory'],
'NPC_run%s' % args['num'].zfill(3),
'CRYSTFEL')
os.chdir(cryst_dir)
h5path = os.path.join((args['output_directory'],
'NPC_run%s' % args['num'].zfill(3), 'HDF5'))
subprocess.call("csh flauncher.csh %s %s *.h5" % (h5path, "%s" %
(args['run'])),
shell=True)
def __init__(self, args):
Client.__init__(self, args)
from NPC_LCLS import ImgReshape, LaserPumpStatus
self.args = args
# LCLS Specific
ds = DataSource('exp=%s:run=%s' % (args.exp, args.run))
det = self.getDetName()
if self.args.TimeResolved:
LaserStatus = LaserPumpStatus()
else: LaserStatus = None
if self.args.SubtractBKG:
if self.args.CrystFELGeom:
from NPC.NPC_routines import ApplyCrystFELGeom
self.getData = ApplyCrystFELGeom
else:
self.getData = det.image
else:
self.getData = det.calib
#Initializing array containing data
img = np.zeros((1480, 1552))
root = ''
#dark = mask = None # Need to open a dark here
for nevent, evt in enumerate(ds.events()):
#if nevent > 10000: break # For testing purpose only
if nevent%(size-1)!=rank-1: continue # different ranks look at different events
self.Ntotal += 1
img2 = det.calib(evt)
if img2 is None:
self.Nerr += 1
continue
# data = self.DataCorrection.DarkCorrection(img2, self.args.roi.tuple) # data shape 32 * 185 * 388
# The hit is performed on the corrected image - using the provided mask
if IsHit(self.DataCorrection.MaskCorrection(img2, self.args.roi.tuple), self.args['threshold'],
self.args['npixels']):
self.Nhits += 1
# TODO: Perform Bragg peak localization
#id = evt.get(EventId).time()
#root = '%s_%s_%i_%i'%(args.exp,str(args.run).zfill(3), id[0], id[1])
#bl_info = evt.get( Bld.BldDataEBeamV7, Source('BldInfo(EBeam)'))
energy = bl_info.ebeamPhotonEnergy()
# OutputFile.create_dataset("energy", data=energy)
self.SaveHits.saveHit(img, root,energy)
#md.addarray((root,), np.array(data), self.Ntotal, self.Nhits, self.Nerr)
# Sending to the master
if self.Ntotal % 10 == 0:
md = mpidata()
md.addarray((root,), np.array(img2), self.Ntotal, self.Nhits, self.Nerr)
md.send()
self.Ntotal = 0
self.Nhits = 0
self.Nerr = 0
self.SaveHits.ClosingOpenH5()
md.endrun() #Should be NPC apply correction function
def runmaster(nClients, args):
t=time.time()
#detector = InitDetector(args)
#max_proj = np.zeros(detector.shape)
Ntotal = 0
Nhits = 0
Nerr = 0
txt = ""
#OutputFileName = os.path.join(args['output_directory'], 'NPC_run%s' % args['num'].zfill(3),
# 'hits_%s' % (args['num'].zfill(3)))
#outTxtHit = open('%s.txt' % OutputFileName, 'w')
#outTxtRej = open('%s_rejected.txt' % OutputFileName, 'w')
md = mpidata()
while nClients > 0:
md.recv()
for arrinfo in md.small.arrayinfolist:
if arrinfo.name != 'maxproj':
try:
data = getattr(md, arrinfo.name[0])
#print "Data shape:", data.shape
Ntotal += arrinfo.Ntotal
Nhits += arrinfo.Nhits
Nerr += arrinfo.Nerr
#txthits, txtnohits = arrinfo.hitsIDX
#if len(txthits) > 0:
# outTxtHit.write(txthits)
#if len(txtnohits) > 0:
# outTxtHit.write(txthits)print >> outTxtRej, txtnohits
print("Processed: %5i - Hits %5i - Errors : %i"%(Ntotal, Nhits, Nerr))#md.small.Ntotal, md.small.Nhits, md
except AttributeError:
print('The data from file %s have not been sent yet to the master'%arrinfo.name[0])
if md.small.endrun:
nClients -= 1
if nClients == 0:
print("Master is done")
tf = time.time()
print('Needed %4.2f s to lookup at %i events with %i workers'%(tf-t, Ntotal, size-1))
if args['crystfel']:
#Should we loop over ranks ??
if args['TimeResolved']:
for laser in ['on', 'off']:
#OutputFilename = os.path.join(self.args['output_directory'],
# 'NPC_run%s' % self.args['num'].zfill(3),
cryst_dir = os.path.join (args['output_directory'], 'NPC_run%s' % args['num'].zfill(3), 'CRYSTFEL_%s' % laser)
#cryst_dir = os.path.join(IO.procdir, IO.H5Dir, 'CRYSTFEL_%s' % laser)
os.mkdir(cryst_dir)
os.chmod(cryst_dir, 777)
# Copy input files to this dir
root_dir = '/home/coquelle/CRYSTFEL_FILES/'
files = ['current.geom', 'current.pdb', 'index.csh', 'stats.csh', 'flauncher.csh']
for f in files:
_f = os.path.join(root_dir, f)
shutil.copy(_f, cryst_dir)
os.chdir(cryst_dir)
h5path = os.path.join(args['output_directory'], 'NPC_run%s' % args['num'].zfill(3), 'HDF5')
subprocess.call(
"csh flauncher.csh %s %s *%s.h5" % (h5path, "%s_%s" % (args['run'], laser), laser),
shell=True)
else:
#cryst_dir = os.path.join(IO.procdir, IO.H5Dir, 'CRYSTFEL')
cryst_dir = os.path.join(args['output_directory'], 'NPC_run%s' % args['num'].zfill(3), 'CRYSTFEL')
os.chdir(cryst_dir)
h5path = os.path.join((args['output_directory'], 'NPC_run%s' % args['num'].zfill(3), 'HDF5'))
subprocess.call("csh flauncher.csh %s %s *.h5" % (h5path, "%s" % (args['run'])), shell=True)