def __init__(self, integration_dirname, out_dirname, event_codes,
alist_names):
"""
@param integration_dirname directory with integration pickle files
@param out_dirname directory for alists
@event_codes space-seperated list of evr event codes
@alist_names corresponding list of names to give the alists
"""
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.setLevel(logging.INFO)
self.integration_dirname = cspad_tbx.getOptString(integration_dirname)
self.out_dirname = cspad_tbx.getOptString(out_dirname)
self.event_codes = cspad_tbx.getOptStrings(event_codes)
self.alist_names = cspad_tbx.getOptStrings(alist_names)
self.event_codes = [int(s) for s in self.event_codes]
# Try to guess multiprocessing method
if 'SGE_TASK_ID' in os.environ and \
'SGE_TASK_FIRST' in os.environ and \
'SGE_TASK_LAST' in os.environ:
if 'SGE_STEP_SIZE' in os.environ:
assert int(os.environ['SGE_STEP_SIZE']) == 1
if os.environ['SGE_TASK_ID'] == 'undefined' or os.environ[
'SGE_TASK_ID'] == 'undefined' or os.environ[
'SGE_TASK_ID'] == 'undefined':
self.rank = 0
self.size = 1
else:
self.rank = int(os.environ['SGE_TASK_ID']) - int(
os.environ['SGE_TASK_FIRST'])
self.size = int(os.environ['SGE_TASK_LAST']) - int(
os.environ['SGE_TASK_FIRST']) + 1
else:
try:
from mpi4py import MPI
except ImportError:
self.rank = 0
self.size = 1
else:
comm = MPI.COMM_WORLD
self.rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
self.size = comm.Get_size(
) # size: number of processes running in this job
# Save a dicitonary of timestamps that satisfy any of the event codes given
self.timestamps_d = {}
for alist in self.alist_names:
self.timestamps_d[alist] = []
def __init__(self,
address,
avg_dirname=None,
avg_basename=None,
stddev_dirname=None,
stddev_basename=None,
max_dirname=None,
max_basename=None,
background_path=None,
flags=None,
hot_threshold=None,
gain_threshold=None,
noise_threshold=7,
elastic_threshold=9,
symnoise_threshold=4,
**kwds):
"""
@param address Full data source address of the DAQ device
@param avg_dirname Directory portion of output average image
XXX mean
@param avg_basename Filename prefix of output average image XXX
mean
@param flags inactive: Eliminate the inactive pixels
noelastic: Eliminate elastic scattering
nohot: Eliminate the hot pixels
nonoise: Eliminate noisy pixels
symnoise: Symmetrically eliminate noisy pixels
@param stddev_dirname Directory portion of output standard
deviation image XXX std
@param stddev_basename Filename prefix of output standard
deviation image XXX std
@param max_dirname Directory portion of output maximum
projection image
@param max_basename Filename prefix of output maximum
projection image
"""
super(average_mixin, self).__init__(
address=address,
**kwds
)
self.roi = None
self.avg_basename = cspad_tbx.getOptString(avg_basename)
self.avg_dirname = cspad_tbx.getOptString(avg_dirname)
self.detector = cspad_tbx.address_split(address)[0]
self.flags = cspad_tbx.getOptStrings(flags, default = [])
self.stddev_basename = cspad_tbx.getOptString(stddev_basename)
self.stddev_dirname = cspad_tbx.getOptString(stddev_dirname)
self.max_basename = cspad_tbx.getOptString(max_basename)
self.max_dirname = cspad_tbx.getOptString(max_dirname)
self.background_path = cspad_tbx.getOptString(background_path)
self.hot_threshold = cspad_tbx.getOptFloat(hot_threshold)
self.gain_threshold = cspad_tbx.getOptFloat(gain_threshold)
self.noise_threshold = cspad_tbx.getOptFloat(noise_threshold)
self.elastic_threshold = cspad_tbx.getOptFloat(elastic_threshold)
self.symnoise_threshold = cspad_tbx.getOptFloat(symnoise_threshold)
if background_path is not None:
background_dict = easy_pickle.load(background_path)
self.background_img = background_dict['DATA']
self._have_max = self.max_basename is not None or \
self.max_dirname is not None
self._have_mean = self.avg_basename is not None or \
self.avg_dirname is not None
self._have_std = self.stddev_basename is not None or \
self.stddev_dirname is not None
# Start a server process which holds a set of Python objects that
# other processes can manipulate using proxies. The queues will
# be used in endjob() to pass images between the worker processes,
# and the lock will ensure the transfer is treated as a critical
# section. There is therefore the risk of a hang if the queues
# cannot hold all the data one process will supply before another
# empties it.
#
# In an attempt to alleviate this issue, separate queues are used
# for the potentially big images. The hope is to prevent
# producers from blocking while consumers are locked out by using
# more buffers.
mgr = multiprocessing.Manager()
self._lock = mgr.Lock()
self._metadata = mgr.dict()
self._queue_max = mgr.Queue()
self._queue_sum = mgr.Queue()
self._queue_ssq = mgr.Queue()
def __init__(self,
address,
dispatch = None,
integration_dirname = None,
integration_basename = None,
out_dirname = None,
out_basename = None,
roi = None,
distl_min_peaks = None,
distl_flags = None,
threshold = None,
xtal_target = None,
negate_hits = False,
trial_id = None,
db_logging = False,
progress_logging = False,
sql_buffer_size = 1,
db_host = None,
db_name = None,
db_table_name = None,
db_experiment_tag = None,
db_user = None,
db_password = None,
db_tags = None,
rungroup_id = None,
**kwds):
"""The mod_hitfind class constructor stores the parameters passed
from the pyana configuration file in instance variables. All
parameters, except @p address are optional, and hence need not be
defined in pyana.cfg.
@param address Full data source address of the DAQ device
@param dispatch Function to call
@param integration_dirname
Directory portion of output integration file
pathname
@param integration_basename
Filename prefix of output integration file
pathname
@param out_dirname Directory portion of output image pathname
@param out_basename Filename prefix of output image pathname
@param roi Region of interest for thresholding, on the
form fast_low:fast_high,slow_low:slow_high
@param threshold Minimum value in region of interest to pass
"""
super(mod_hitfind, self).__init__(address=address, **kwds)
self.m_dispatch = cspad_tbx.getOptString(dispatch)
self.m_integration_basename = cspad_tbx.getOptString(integration_basename)
self.m_integration_dirname = cspad_tbx.getOptString(integration_dirname)
self.m_out_basename = cspad_tbx.getOptString(out_basename)
self.m_out_dirname = cspad_tbx.getOptString(out_dirname)
self.m_distl_min_peaks = cspad_tbx.getOptInteger(distl_min_peaks)
self.m_distl_flags = cspad_tbx.getOptStrings(distl_flags)
self.m_threshold = cspad_tbx.getOptInteger(threshold)
self.m_xtal_target = cspad_tbx.getOptString(xtal_target)
self.m_negate_hits = cspad_tbx.getOptBool(negate_hits)
self.m_trial_id = cspad_tbx.getOptInteger(trial_id)
self.m_db_logging = cspad_tbx.getOptBool(db_logging)
self.m_progress_logging = cspad_tbx.getOptBool(progress_logging)
self.m_sql_buffer_size = cspad_tbx.getOptInteger(sql_buffer_size)
self.m_db_host = cspad_tbx.getOptString(db_host)
self.m_db_name = cspad_tbx.getOptString(db_name)
self.m_db_table_name = cspad_tbx.getOptString(db_table_name)
self.m_db_experiment_tag = cspad_tbx.getOptString(db_experiment_tag)
self.m_db_user = cspad_tbx.getOptString(db_user)
self.m_db_password = cspad_tbx.getOptString(db_password)
self.m_db_tags = cspad_tbx.getOptString(db_tags)
self.m_rungroup_id = cspad_tbx.getOptInteger(rungroup_id)
# A ROI should not contain any ASIC boundaries, as these are
# noisy. Hence circular ROI:s around the beam centre are probably
# not such a grand idea.
self.m_roi = cspad_tbx.getOptROI(roi)
# Verify that dist_min_peaks is either "restrictive" or
# "permissive", but not both. ^ is the logical xor operator
if self.m_distl_min_peaks is not None:
if (not (('permissive' in self.m_distl_flags) ^
('restrictive' in self.m_distl_flags))):
raise RuntimeError("""Sorry, with the distl_min_peaks option,
distl_flags must be set to 'permissive' or 'restrictive'.""")
if (self.m_roi is not None):
raise RuntimeError("""Sorry, either specify region of interest
(roi) or distl_min_peaks, but not both.""")
if self.m_db_logging:
self.buffered_sql_entries = []
assert self.m_sql_buffer_size >= 1
if self.m_progress_logging:
self.buffered_progress_entries = []
assert self.m_sql_buffer_size >= 1
self.isoforms = {}
if self.m_db_tags is None:
self.m_db_tags = ""
def __init__(self,
address,
dispatch=None,
integration_dirname=None,
integration_basename=None,
out_dirname=None,
out_basename=None,
roi=None,
distl_min_peaks=None,
distl_flags=None,
threshold=None,
xtal_target=None,
negate_hits=False,
trial_id=None,
db_logging=False,
progress_logging=False,
sql_buffer_size=1,
db_host=None,
db_name=None,
db_table_name=None,
db_experiment_tag=None,
db_user=None,
db_password=None,
db_tags=None,
trial=None,
rungroup_id=None,
db_version='v1',
**kwds):
"""The mod_hitfind class constructor stores the parameters passed
from the pyana configuration file in instance variables. All
parameters, except @p address are optional, and hence need not be
defined in pyana.cfg.
@param address Full data source address of the DAQ device
@param dispatch Function to call
@param integration_dirname
Directory portion of output integration file
pathname
@param integration_basename
Filename prefix of output integration file
pathname
@param out_dirname Directory portion of output image pathname
@param out_basename Filename prefix of output image pathname
@param roi Region of interest for thresholding, on the
form fast_low:fast_high,slow_low:slow_high
@param threshold Minimum value in region of interest to pass
"""
super(mod_hitfind, self).__init__(address=address, **kwds)
self.m_dispatch = cspad_tbx.getOptString(dispatch)
self.m_integration_basename = cspad_tbx.getOptString(
integration_basename)
self.m_integration_dirname = cspad_tbx.getOptString(
integration_dirname)
self.m_out_basename = cspad_tbx.getOptString(out_basename)
self.m_out_dirname = cspad_tbx.getOptString(out_dirname)
self.m_distl_min_peaks = cspad_tbx.getOptInteger(distl_min_peaks)
self.m_distl_flags = cspad_tbx.getOptStrings(distl_flags)
self.m_threshold = cspad_tbx.getOptInteger(threshold)
self.m_xtal_target = cspad_tbx.getOptString(xtal_target)
self.m_negate_hits = cspad_tbx.getOptBool(negate_hits)
self.m_trial_id = cspad_tbx.getOptInteger(trial_id)
self.m_db_logging = cspad_tbx.getOptBool(db_logging)
self.m_progress_logging = cspad_tbx.getOptBool(progress_logging)
self.m_sql_buffer_size = cspad_tbx.getOptInteger(sql_buffer_size)
self.m_db_host = cspad_tbx.getOptString(db_host)
self.m_db_name = cspad_tbx.getOptString(db_name)
self.m_db_table_name = cspad_tbx.getOptString(db_table_name)
self.m_db_experiment_tag = cspad_tbx.getOptString(db_experiment_tag)
self.m_db_user = cspad_tbx.getOptString(db_user)
self.m_db_password = cspad_tbx.getOptString(db_password)
self.m_db_tags = cspad_tbx.getOptString(db_tags)
self.m_trial = cspad_tbx.getOptInteger(trial)
self.m_rungroup_id = cspad_tbx.getOptInteger(rungroup_id)
self.m_db_version = cspad_tbx.getOptString(db_version)
# A ROI should not contain any ASIC boundaries, as these are
# noisy. Hence circular ROI:s around the beam centre are probably
# not such a grand idea.
self.m_roi = cspad_tbx.getOptROI(roi)
# Verify that dist_min_peaks is either "restrictive" or
# "permissive", but not both. ^ is the logical xor operator
if self.m_distl_min_peaks is not None:
if (not (('permissive' in self.m_distl_flags) ^
('restrictive' in self.m_distl_flags))):
raise RuntimeError("""Sorry, with the distl_min_peaks option,
distl_flags must be set to 'permissive' or 'restrictive'.""")
if (self.m_roi is not None):
raise RuntimeError("""Sorry, either specify region of interest
(roi) or distl_min_peaks, but not both.""")
if self.m_db_logging:
self.buffered_sql_entries = []
assert self.m_sql_buffer_size >= 1
if self.m_progress_logging:
if self.m_db_version == 'v1':
self.buffered_progress_entries = []
assert self.m_sql_buffer_size >= 1
self.isoforms = {}
elif self.m_db_version == 'v2':
from xfel.ui import db_phil_str
from libtbx.phil import parse
extra_phil = """
input {
trial = None
.type = int
trial_id = None
.type = int
rungroup = None
.type = int
}
"""
self.db_params = parse(db_phil_str + extra_phil).extract()
self.db_params.experiment_tag = self.m_db_experiment_tag
self.db_params.db.host = self.m_db_host
self.db_params.db.name = self.m_db_name
self.db_params.db.user = self.m_db_user
self.db_params.db.password = self.m_db_password
self.db_params.input.trial = self.m_trial
self.db_params.input.rungroup = self.m_rungroup_id
if self.m_db_tags is None:
self.m_db_tags = ""
def __init__(self,
address,
avg_dirname=None,
avg_basename=None,
stddev_dirname=None,
stddev_basename=None,
max_dirname=None,
max_basename=None,
background_path=None,
flags=None,
hot_threshold=None,
gain_threshold=None,
noise_threshold=7,
elastic_threshold=9,
symnoise_threshold=4,
**kwds):
"""
@param address Full data source address of the DAQ device
@param avg_dirname Directory portion of output average image
XXX mean
@param avg_basename Filename prefix of output average image XXX
mean
@param flags inactive: Eliminate the inactive pixels
noelastic: Eliminate elastic scattering
nohot: Eliminate the hot pixels
nonoise: Eliminate noisy pixels
symnoise: Symmetrically eliminate noisy pixels
@param stddev_dirname Directory portion of output standard
deviation image XXX std
@param stddev_basename Filename prefix of output standard
deviation image XXX std
@param max_dirname Directory portion of output maximum
projection image
@param max_basename Filename prefix of output maximum
projection image
"""
super(average_mixin, self).__init__(
address=address,
**kwds
)
self.roi = None
self.avg_basename = cspad_tbx.getOptString(avg_basename)
self.avg_dirname = cspad_tbx.getOptString(avg_dirname)
self.detector = cspad_tbx.address_split(address)[0]
self.flags = cspad_tbx.getOptStrings(flags, default = [])
self.stddev_basename = cspad_tbx.getOptString(stddev_basename)
self.stddev_dirname = cspad_tbx.getOptString(stddev_dirname)
self.max_basename = cspad_tbx.getOptString(max_basename)
self.max_dirname = cspad_tbx.getOptString(max_dirname)
self.background_path = cspad_tbx.getOptString(background_path)
self.hot_threshold = cspad_tbx.getOptFloat(hot_threshold)
self.gain_threshold = cspad_tbx.getOptFloat(gain_threshold)
self.noise_threshold = cspad_tbx.getOptFloat(noise_threshold)
self.elastic_threshold = cspad_tbx.getOptFloat(elastic_threshold)
self.symnoise_threshold = cspad_tbx.getOptFloat(symnoise_threshold)
if background_path is not None:
background_dict = easy_pickle.load(background_path)
self.background_img = background_dict['DATA']
self._have_max = self.max_basename is not None or \
self.max_dirname is not None
self._have_mean = self.avg_basename is not None or \
self.avg_dirname is not None
self._have_std = self.stddev_basename is not None or \
self.stddev_dirname is not None
# Start a server process which holds a set of Python objects that
# other processes can manipulate using proxies. The queues will
# be used in endjob() to pass images between the worker processes,
# and the lock will ensure the transfer is treated as a critical
# section. There is therefore the risk of a hang if the queues
# cannot hold all the data one process will supply before another
# empties it.
#
# In an attempt to alleviate this issue, separate queues are used
# for the potentially big images. The hope is to prevent
# producers from blocking while consumers are locked out by using
# more buffers.
mgr = multiprocessing.Manager()
self._lock = mgr.Lock()
self._metadata = mgr.dict()
self._queue_max = mgr.Queue()
self._queue_sum = mgr.Queue()
self._queue_ssq = mgr.Queue()