def _get_evt_det(self, evt):
if self.acq_num == 1:
acq1_src = psana.Source('DetInfo(SxrEndstation.0:Acqiris.1)')
acq_det = evt.get(psana.Acqiris.DataDescV1, acq1_src)
elif self.acq_num == 2:
acq2_src = psana.Source('DetInfo(SxrEndstation.0:Acqiris.2)')
acq_det = evt.get(psana.Acqiris.DataDescV1, acq2_src)
return acq_det
def __init__(self):
#Handle warnings
warnings.filterwarnings('always',module='Utils',category=UserWarning)
warnings.filterwarnings('ignore',module='Utils',category=RuntimeWarning, message="invalid value encountered in divide")
#Some default values for the options
self._experiment='amoc8114' #Experiment label
self._darkreferencepath=[]; #Dark reference information
self._lasingoffreferencepath=[]; #Dark reference information
self._darkreference=[]
self._lasingoffreference=[]
self._nb=float('nan') #Number of bunches
self._medianfilter=float('nan') #Number of neighbours for median filter
self._snrfilter=float('nan') #Number of sigmas for the noise threshold
self._roiwaistthres=float('nan') #Parameter for the roi location
self._roiexpand=float('nan') #Parameter for the roi location
self._islandsplitmethod='' #Method for island splitting
self._islandsplitpar1=float('nan')
self._islandsplitpar2=float('nan')
self._currentevent=[]
self._eventresultsstep1=[]
self._eventresultsstep2=[]
self._eventresultsstep3=[]
self._env=[]
self._globalcalibration=[]
self._roixtcav=[]
self._calpath=''
#Different flags
self._loadeddarkreference=False
self._loadedlasingoffreference=False
self._currenteventavailable=False
self._currenteventprocessedstep1=False #Step one is pure processing of the trace, without lasing off referencing
self._currenteventprocessedstep2=False #Step two is calculating physical units
self._currenteventprocessedstep3=False #Step three is the processing of the profiles with respect to the reference
self._envinfo=False
#Camera and type for the xtcav images
self.xtcav_camera = psana.Source('DetInfo(XrayTransportDiagnostic.0:Opal1000.0)')
self.xtcav_type=psana.Camera.FrameV1
self._rawimage=[]
#Ebeam type: it should actually be the version 5 which is the one that contains xtcav stuff
self.ebeam_data=psana.Source('BldInfo(EBeam)')
self._ebeam=[]
#Gas detectors for the pulse energies
self.gasdetector_data=psana.Source('BldInfo(FEEGasDetEnergy)')
self._gasdetector=[]
def get_sharpness_vs_position():
'''
This is one function because I could not figure out how to get it to work as separate functions
'''
ds = psana.DataSource("exp=SXR/sxri0214:run=139:idx")
src = psana.Source("DetInfo(SxrBeamline.0:Opal1000.1)")
epics = ds.env().epicsStore()
m1_positions = []
sharp_vals = []
print ds.runs()
for run in ds.runs(): #new #maybe there is only one run? It prints in a
#way I don't understand
print run
times = run.times()
#inxs = []
#indx = 0
for tm in times[0:120000]:
print tm
evt = run.event(tm)
camera = evt.get(psana.Camera.FrameV1, src)
projection = np.sum(camera.data16(), axis=1)
a_val = working_algorithm.main(projection)
sharp_vals.append(a_val)
#if a_val < 100:
# inxs.append(indx)
position = epics.value('SXR:MON:MMS:05.RBV')
m1_positions.append(position)
#indx += 1
ar = np.array([sharp_vals, m1_positions])
print ar
return ar
def look_at_spectrometer():
plt.figure(1)
for run in range(282, 291):
runstr = 'exp=diamcc14:run=%d' % run
try:
ds = psana.DataSource(runstr)
except Exception, exp:
print("unable to open %s" % runstr)
print(exp)
continue
for evt in ds.events():
opal = evt.get(
psana.Camera.FrameV1,
psana.Source('DetInfo(XrayTransportDiagnostic.20:Opal1000.0)'))
if None is opal: continue
MX = 500
plt.imshow(np.minimum(
MX, np.maximum(0,
opal.data16().astype(np.float))),
vmin=0,
vmax=MX,
interpolation='none')
plt.show()
plt.pause(.1)
raw_input('hi')
def acqiris_data_dataext(event):
return event['evt'].get(
psana.Acqiris.DataDescV1,
psana.Source(gp.monitor_params['PsanaParallelizationLayer']
['digitizer_psana_source'])).data(
gp.monitor_params['PsanaParallelizationLayer']
['digitizer_psana_channel']).waveforms()
def get_rayonix_detector_dimensions(env):
''' Given a psana env object, find the detector dimensions
@param env psana environment object
'''
import psana
cfgs = env.configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source('Rayonix'))
return rayonix_cfg.width(), rayonix_cfg.height()
def psana_event_inspection(source):
"""Prints the structure of psana events.
Takes a psana source string (e.g. exp=CXI/cxix....) and inspects the
structure of the first event in the data, printing information about
the the content of the event.
Args:
source (str): a psana source string (e.g. exp=CXI/cxix....).
"""
import psana
def my_import(name):
mod = __import__(name)
components = name.split('.')
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
def my_psana_from_string(name):
components = name.split('.')
mod = __import__(components[0])
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
print('\n\n')
print('data source : %s' % source)
print('\n')
print('Opening dataset...')
ds = psana.DataSource(source)
print('\n')
print('Epics pv Names (the confusing ones):')
print(ds.env().epicsStore().pvNames())
print('\n')
print('Epics aliases (the not so confusing ones):')
print(ds.env().epicsStore().aliases())
print('\n')
print('Event structure:')
itr = ds.events()
evt = itr.next()
for k in evt.keys():
print('Type: %s Source: %s Alias: %s Key: %s') % (
k.type(), k.src(), k.alias(), k.key())
print('\n')
for k in evt.keys():
print(k)
beam = evt.get(psana.Bld.BldDataEBeamV7, psana.Source('BldInfo(EBeam)'))
print('Photon energy: %f' % beam.ebeamPhotonEnergy())
def print_evrs(run_num):
ds = psana.DataSource('exp=cxif7214:run=39')
for evt in ds.events():
print[
e.eventCode() for e in evt.get(
psana.EvrData.DataV3,
psana.Source('DetInfo(NoDetector.0:Evr.0)')).fifoEvents()
]
return
def data(self, evt):
dl={}
try:
l3t = evt.get(psana.L3T.DataV2,psana.Source(''))
if l3t is not None:
dl['accept']=l3t.accept()
except:
pass
return dl
def pump_laser_state_dataext(event):
evr = event['evt'].get(psana.EvrData.DataV4,
psana.Source('DetInfo(NoDetector.0:Evr.1)'))
if not evr:
evr = event['evt'].get(psana.EvrData.DataV4,
psana.Source('DetInfo(NoDetector.0:Evr.2)'))
if not evr.present(163):
if evr.present(
gp.monitor_params['RadialAveraging']['pump_laser_evr_code']):
#pump laser on
return 1
else:
#pump laser off
return 0
else:
#dropped shot
return 2
def psana_source(env, srcpar):
"""returns psana.Source(src) from other psana.Source brief src or alias.
Parameters
- srcpar : str - regular source or its alias, ex.: 'XppEndstation.0:Rayonix.0' or 'rayonix'
- set_sub : bool - default=True - propagates source parameter to low level package
"""
#print 'type of srcpar: ', type(srcpar)
src = srcpar if isinstance(srcpar, psana.Source) else psana.Source(srcpar)
str_src = string_from_source(src)
amap = env.aliasMap()
psasrc = amap.src(str_src)
source = src if amap.alias(psasrc) == '' else amap.src(str_src)
if not isinstance(source, psana.Source): source = psana.Source(source)
return source
def get_source(source_string, verbose=False):
if verbose:
print 'In "simplepsana.get_source()" function.'
if psana is None:
print 'ERROR: Function "simplepsana.getSoutrce" cannot be used without psana.'
sys.exit()
global _sourceDict
if source_string not in _sourceDict:
_sourceDict[source_string] = psana.Source(source_string)
return _sourceDict[source_string]
def __init__(self, detname, name=None, saveTime=False):
if name is None:
self.name = detname
else:
self.name = name
self.saveTime = saveTime
defaultDetector.__init__(self, detname, self.name)
cfg = self.det.env.configStore().get(psana.Generic1D.ConfigV0, psana.Source(detname))
self.wave8_shape = None
if cfg is not None:
self.wave8_shape = cfg.Length()
def __init__(self, detname, name=None, saveTime=False):
if name is None:
self.name = detname
else:
self.name = name
self.saveTime = saveTime
defaultDetector.__init__(self, detname, self.name)
cfg = self.det.env.configStore().get(psana.Imp.ConfigV1, psana.Source(detname))
self.imp_shape = None
if cfg is not None:
self.imp_shape = cfg.numberOfSamples()
def _retrieve_cspad_evt_data(self, evt):
"""
Retrieve intensities collected on CsPad from input event.
:param evt: psana event object
:return data: list of quads that make up measured data
"""
data = list()
cspad = evt.get(psana.CsPad.DataV2, psana.Source('DscCsPad'))
for num in range(cspad.quads_shape()[0]):
data.append(cspad.quads(num).data())
return data
def __init__(self, params):
self.src = psana.Source(params.spectra_filter.detector_address)
self.roi = params.spectra_filter.roi
self.bg_roi = params.spectra_filter.background_roi
if params.spectra_filter.background_path is None:
self.dark_pickle = None
else:
self.dark_pickle = easy_pickle.load(
params.spectra_filter.background_path)['DATA'].as_numpy_array(
)
self.peak_range = params.spectra_filter.peak_range
self.params = params
def __init__(self, image_file, **kwargs):
super().__init__(image_file, locator_scope=rayonix_locator_scope, **kwargs)
cfgs = self._ds.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix"))
if self.params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
bin_size = rayonix_cfg.binning_f()
else:
bin_size = self.params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size)
self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(self._ds.env())
def __init__(self, config, quiet=True):
self._evt = None
# EBeam
self._EBeamSource = psana.Source('BldInfo(EBeam)')
self._EBeamType = None
self._phaseCavitySource = psana.Source('BldInfo(PhaseCavity)')
# Evr
self._EvrSource = psana.Source('DetInfo(NoDetector.0:Evr.0)')
self._EvrType = psana.EvrData.DataV3
# photon beam
#photonCalib = loadConfig(calibFile, quiet=quiet)
#self._photonA = photonCalib.A
#self._photonB = photonCalib.B
self._photonA = config.lcls_photonEnergyA
self._photonB = config.lcls_photonEnergyB
self._feeSource = psana.Source('BldInfo(FEEGasDetEnergy)')
self._idData = None
def test_detector_full_name():
ds = psana.DataSource(
'/reg/g/psdm/detector/data_test/types/0007-NoDetector.0-Epix100a.0.xtc'
)
env = ds.env()
print 20 * '_', '\n%s:' % sys._getframe().f_code.co_name
print 'src="Epix" :', detector_full_name(
env, "Epix")
print 'src=psana.Source("Epix")) :', detector_full_name(
env, psana.Source('Epix'))
print 'src="DetInfo(NoDetector.0:Epix100a.0)":', detector_full_name(
env, 'DetInfo(NoDetector.0:Epix100a.0)')
print 'for alias src="cs140_0" :', detector_full_name(
env, 'cs140_0')
def __init__(self, params, run, detector, **kwargs):
from xfel.cxi.cspad_ana import rayonix_tbx
import psana
FormatXTCSingleEvent.__init__(self, params, run, detector, **kwargs)
self._cached_detector = {}
cfgs = run.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix"))
if params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
bin_size = rayonix_cfg.binning_f()
else:
bin_size = params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size)
self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(
run.env())
def getEventID(evt):
try:
# timestamp
evtid = evt.get(psana.EventId)
seconds = evtid.time()[0]
nanoseconds = evtid.time()[1]
fiducials = evtid.fiducials()
# photon energy
ebeam = evt.get(psana.Bld.BldDataEBeamV7,
psana.Source('BldInfo(EBeam)'))
photonEnergy = ebeam.ebeamPhotonEnergy()
return seconds, nanoseconds, fiducials, photonEnergy
except:
pass
return None, None, None, None
def __init__(self, image_file, **kwargs):
super(FormatXTCRayonix, self).__init__(
image_file, locator_scope=rayonix_locator_scope, **kwargs
)
self._ds = FormatXTCRayonix._get_datasource(image_file, self.params)
self._env = self._ds.env()
self.populate_events()
self.n_images = len(self.times)
cfgs = self._ds.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix"))
if self.params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
bin_size = rayonix_cfg.binning_f()
else:
bin_size = self.params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size)
self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(self._ds.env())
def main():
args = parse_args()
if args.run == 0:
print 'Analyzing data from shared memory...'
try:
ds = psana.DataSource('shmem=1_1_psana_XCS.0:stop=no')
except:
raise IOError('Cannot find shared memory stream.')
else:
print 'Analyzing run: %d' % args.run
ds = psana.DataSource('exp=sxra8513:run=%d' %
args.run) # CHANGE THIS FOR NEW EXPT
# this may also need to change for the new expt
opal_src = psana.Source('DetInfo(SxrEndstation.0:Opal1000.1)')
# iterate over events and extract peaks
for i, evt in enumerate(ds.events()):
# gets an "opal" object
opal = evt.get(psana.Camera.FrameV1, opal_src)
if opal:
image = opal.data16().copy(
) # this is a numpy array of the opal image
centers, widths = find_blobs(image, sigma_threshold=args.sigma)
n_blobs = len(centers)
print 'Shot %d :: found %d blobs :: %s' % (i, n_blobs,
str(centers))
if args.view and (n_blobs > 0):
draw_blobs(image, centers, widths)
# if we reach the max number of shots, stop
if i + 1 == args.num_max:
print 'Reached max requested number of shots (%d)' % (i + 1, )
break
return
def __init__(self, image_file, **kwargs):
import psana
assert (self.understand(image_file))
FormatXTC.__init__(self,
image_file,
locator_scope=rayonix_locator_scope,
**kwargs)
self._ds = FormatXTCRayonix._get_datasource(image_file, self.params)
self._env = self._ds.env()
self.populate_events()
self.n_images = len(self.times)
cfgs = self._ds.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source('Rayonix'))
if self.params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
self._bin_size = rayonix_cfg.binning_f()
else:
self._bin_size = self.params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(self._bin_size)
self._image_size = rayonix_cfg.width(), rayonix_cfg.height()
def updatePhotonEnergy(self, arg):
if arg == 'lcls':
self.parent.ebeam = self.parent.evt.get(
psana.Bld.BldDataEBeamV7, psana.Source('BldInfo(EBeam)'))
if self.parent.ebeam:
self.parent.photonEnergy = self.parent.ebeam.ebeamPhotonEnergy(
)
else:
try:
wavelength = self.parent.epics.value(
'SIOC:SYS0:ML00:AO192') # nanometre
h = 6.626070e-34 # J.m
c = 2.99792458e8 # m/s
joulesPerEv = 1.602176621e-19 # J/eV
self.parent.photonEnergy = (h / joulesPerEv *
c) / (wavelength * 1e-9)
except:
self.parent.photonEnergy = 0.0
self.parent.geom.p1.param(
self.parent.geom.geom_grp,
self.parent.geom.geom_photonEnergy_str).setValue(
self.parent.photonEnergy)
else:
cspad_data_shape = None
# print(cspad_data_shape)
comm.Barrier()
cspad_data_shape = comm.bcast(cspad_data_shape, root=0)
####
ds = psana.MPIDataSource('exp=' + expname + ':run=%s:smd'%run_num)
# psana.DataSource('dir=/reg/d/ffb/xpp/xpph6615/xtc/:exp=xpph6615:run=%s:idx'%run)
epics = ds.env().epicsStore()
configStore = ds.env().configStore()
total_events = 0
cspadDet = psana.Detector('jungfrau1M',ds.env())
ipm2_src = psana.Source('BldInfo(XPP-SB2-BMMON)')
ipm3_src = psana.Source('BldInfo(XPP-SB3-BMMON)')
evr_src = psana.Source("DetInfo(NoDetector.0:Evr.0)")
def mpi_message(msg):
if rank == 0:
print(msg)
""" Filters by skipping events that are outside of the desired range. Comment out what you do not want.
"""
def filter_events(evts):
skipctr = 0
count = 0
for ev in evts:
count += 1
def run(self):
""" Process all images assigned to this thread """
params, options = self.parser.parse_args(show_diff_phil=True)
if params.input.experiment is None or \
params.input.run_num is None or \
params.input.address is None:
raise Usage(self.usage)
if params.format.file_format == "cbf":
if params.format.cbf.detz_offset is None:
raise Usage(self.usage)
elif params.format.file_format == "pickle":
if params.format.pickle.cfg is None:
raise Usage(self.usage)
else:
raise Usage(self.usage)
if not os.path.exists(params.output.output_dir):
raise Sorry("Output path not found:" + params.output.output_dir)
# Save the paramters
self.params = params
self.options = options
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
size = comm.Get_size() # size: number of processes running in this job
# set up psana
if params.format.file_format == "pickle":
psana.setConfigFile(params.format.pickle.cfg)
dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment,
params.input.run_num)
ds = psana.DataSource(dataset_name)
if params.format.file_format == "cbf":
src = psana.Source('DetInfo(%s)' % params.input.address)
psana_det = psana.Detector(params.input.address, ds.env())
# set this to sys.maxint to analyze all events
if params.dispatch.max_events is None:
max_events = sys.maxint
else:
max_events = params.dispatch.max_events
for run in ds.runs():
if params.format.file_format == "cbf":
# load a header only cspad cbf from the slac metrology
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, params.input.address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
# list of all events
times = run.times()
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [
times[i] for i in xrange(nevents) if (i + rank) % size == 0
]
for i in xrange(len(mytimes)):
evt = run.event(mytimes[i])
id = evt.get(psana.EventId)
print "Event #", i, " has id:", id
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if timestamp is None:
print "No timestamp, skipping shot"
continue
t = timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[
17:19] + t[20:23]
print "Processing shot", s
if params.format.file_format == "pickle":
if evt.get("skip_event"):
print "Skipping event", id
continue
# the data needs to have already been processed and put into the event by psana
data = evt.get(params.format.pickle.out_key)
if data is None:
print "No data"
continue
# set output paths according to the templates
path = os.path.join(params.output.output_dir,
"shot-" + s + ".pickle")
print "Saving", path
easy_pickle.dump(path, data)
elif params.format.file_format == "cbf":
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=params.format.cbf.gain_mask_value
is not None,
gain_mask_value=params.format.cbf.gain_mask_value,
per_pixel_gain=False)
distance = cspad_tbx.env_distance(
params.input.address, run.env(),
params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
continue
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
continue
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx, data, distance, wavelength, timestamp,
params.input.address)
path = os.path.join(params.output.output_dir,
"shot-" + s + ".cbf")
print "Saving", path
# write the file
import pycbf
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
run.end()
ds.end()
f = h5py.File('/reg/neh/home2/tjlane/analysis/xppb0114/averages/r262_laser_avg.h5')
vacuum = (np.array(f['/laser_on']) + np.array(f['/laser_off'])) / 2.0
f.close()
# Define experiment, run. Shared memory syntax (for SXR): shmem=0_1_SXR.0:stop=no
if args.run >= 0:
print "\nReading run #: %d" % args.run
ds = psana.DataSource('exp=xppb0114:run=%d' % args.run)
elif args.run == -1:
print "\nLocking on to shared memory..."
ds = psana.DataSource('shmem=1_1_psana_XPP.0:stop=no')
else:
raise ValueError('`run` parameter must either be an int corresponding to a '
'run, or the keyword "online"')
cspad_src = psana.Source('DetInfo(XppGon.0:Cspad.0)')
evr_src = psana.Source('DetInfo(NoDetector.0:Evr.0)')
update_frequency = 100
# ------ END CONFIG ---------
epics = ds.env().epicsStore()
calib = ds.env().calibStore()
laser_on = np.zeros((32, 185, 194*2))
laser_off = np.zeros((32, 185, 194*2))
shot_index = 0
def run(args):
user_phil = []
for arg in args:
if os.path.isfile(arg):
try:
user_phil.append(parse(file_name=arg))
except Exception as e:
print(str(e))
raise Sorry("Couldn't parse phil file %s" % arg)
else:
try:
user_phil.append(parse(arg))
except Exception as e:
print(str(e))
raise Sorry("Couldn't parse argument %s" % arg)
params = phil_scope.fetch(sources=user_phil).extract()
# cxid9114, source fee: FeeHxSpectrometer.0:Opal1000.1, downstream: CxiDg3.0:Opal1000.0
# cxig3614, source fee: FeeHxSpectrometer.0:OrcaFl40.0
src = psana.Source(params.spectra_filter.detector_address)
dataset_name = "exp=%s:run=%s:idx" % (params.experiment, params.runs)
print("Dataset string:", dataset_name)
ds = psana.DataSource(dataset_name)
spf = spectra_filter(params)
if params.selected_filter == None:
filter_name = params.spectra_filter.filter[0].name
else:
filter_name = params.selected_filter
rank = 0
size = 1
max_events = sys.maxsize
for run in ds.runs():
print("starting run", run.run())
# list of all events
times = run.times()
if params.skip_events is not None:
times = times[params.skip_events:]
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in range(nevents) if (i + rank) % size == 0]
for i, t in enumerate(mytimes):
evt = run.event(t)
accepted, data, spectrum, dc_offset, all_data, all_data_raw = spf.filter_event(
evt, filter_name)
if not accepted:
continue
print(cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)),
"Publishing data for event", i)
#header = "Event %d, m/f: %7.7f, f: %d"%(i, peak_max/flux, flux)
header = "Event %d" % (i)
if rank == 0:
fee = Image(header, "FEE", data) # make a 2D plot
publish.send("FEE", fee) # send to the display
spectrumplot = XYPlot(header, 'summed 1D trace',
list(range(data.shape[1])),
spectrum) # make a 1D plot
publish.send("SPECTRUM", spectrumplot) # send to the display
fee = Image(header, "DC_OFFSET", dc_offset) # make a 2D plot
publish.send("DC_OFFSET", fee) # send to the display
fee = Image(header, "ALL_FEE", all_data) # make a 2D plot
publish.send("ALL_FEE", fee) # send to the display
fee = Image(header, "ALL_FEE_RAW",
all_data_raw) # make a 2D plot
publish.send("ALL_FEE_RAW", fee) # send to the display
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
