def __init__(self, integration_dirname, out_dirname, event_codes, alist_names): """ @param integration_dirname directory with integration pickle files @param out_dirname directory for alists @event_codes space-seperated list of evr event codes @alist_names corresponding list of names to give the alists """ self.logger = logging.getLogger(self.__class__.__name__) self.logger.setLevel(logging.INFO) self.integration_dirname = cspad_tbx.getOptString(integration_dirname) self.out_dirname = cspad_tbx.getOptString(out_dirname) self.event_codes = cspad_tbx.getOptStrings(event_codes) self.alist_names = cspad_tbx.getOptStrings(alist_names) self.event_codes = [int(s) for s in self.event_codes] # Try to guess multiprocessing method if 'SGE_TASK_ID' in os.environ and \ 'SGE_TASK_FIRST' in os.environ and \ 'SGE_TASK_LAST' in os.environ: if 'SGE_STEP_SIZE' in os.environ: assert int(os.environ['SGE_STEP_SIZE']) == 1 if os.environ['SGE_TASK_ID'] == 'undefined' or os.environ[ 'SGE_TASK_ID'] == 'undefined' or os.environ[ 'SGE_TASK_ID'] == 'undefined': self.rank = 0 self.size = 1 else: self.rank = int(os.environ['SGE_TASK_ID']) - int( os.environ['SGE_TASK_FIRST']) self.size = int(os.environ['SGE_TASK_LAST']) - int( os.environ['SGE_TASK_FIRST']) + 1 else: try: from mpi4py import MPI except ImportError: self.rank = 0 self.size = 1 else: comm = MPI.COMM_WORLD self.rank = comm.Get_rank( ) # each process in MPI has a unique id, 0-indexed self.size = comm.Get_size( ) # size: number of processes running in this job # Save a dicitonary of timestamps that satisfy any of the event codes given self.timestamps_d = {} for alist in self.alist_names: self.timestamps_d[alist] = []
def __init__(self, address, avg_dirname=None, avg_basename=None, stddev_dirname=None, stddev_basename=None, max_dirname=None, max_basename=None, background_path=None, flags=None, hot_threshold=None, gain_threshold=None, noise_threshold=7, elastic_threshold=9, symnoise_threshold=4, **kwds): """ @param address Full data source address of the DAQ device @param avg_dirname Directory portion of output average image XXX mean @param avg_basename Filename prefix of output average image XXX mean @param flags inactive: Eliminate the inactive pixels noelastic: Eliminate elastic scattering nohot: Eliminate the hot pixels nonoise: Eliminate noisy pixels symnoise: Symmetrically eliminate noisy pixels @param stddev_dirname Directory portion of output standard deviation image XXX std @param stddev_basename Filename prefix of output standard deviation image XXX std @param max_dirname Directory portion of output maximum projection image @param max_basename Filename prefix of output maximum projection image """ super(average_mixin, self).__init__( address=address, **kwds ) self.roi = None self.avg_basename = cspad_tbx.getOptString(avg_basename) self.avg_dirname = cspad_tbx.getOptString(avg_dirname) self.detector = cspad_tbx.address_split(address)[0] self.flags = cspad_tbx.getOptStrings(flags, default = []) self.stddev_basename = cspad_tbx.getOptString(stddev_basename) self.stddev_dirname = cspad_tbx.getOptString(stddev_dirname) self.max_basename = cspad_tbx.getOptString(max_basename) self.max_dirname = cspad_tbx.getOptString(max_dirname) self.background_path = cspad_tbx.getOptString(background_path) self.hot_threshold = cspad_tbx.getOptFloat(hot_threshold) self.gain_threshold = cspad_tbx.getOptFloat(gain_threshold) self.noise_threshold = cspad_tbx.getOptFloat(noise_threshold) self.elastic_threshold = cspad_tbx.getOptFloat(elastic_threshold) self.symnoise_threshold = cspad_tbx.getOptFloat(symnoise_threshold) if background_path is not None: background_dict = easy_pickle.load(background_path) self.background_img = background_dict['DATA'] self._have_max = self.max_basename is not None or \ self.max_dirname is not None self._have_mean = self.avg_basename is not None or \ self.avg_dirname is not None self._have_std = self.stddev_basename is not None or \ self.stddev_dirname is not None # Start a server process which holds a set of Python objects that # other processes can manipulate using proxies. The queues will # be used in endjob() to pass images between the worker processes, # and the lock will ensure the transfer is treated as a critical # section. There is therefore the risk of a hang if the queues # cannot hold all the data one process will supply before another # empties it. # # In an attempt to alleviate this issue, separate queues are used # for the potentially big images. The hope is to prevent # producers from blocking while consumers are locked out by using # more buffers. mgr = multiprocessing.Manager() self._lock = mgr.Lock() self._metadata = mgr.dict() self._queue_max = mgr.Queue() self._queue_sum = mgr.Queue() self._queue_ssq = mgr.Queue()
def __init__(self, address, dispatch = None, integration_dirname = None, integration_basename = None, out_dirname = None, out_basename = None, roi = None, distl_min_peaks = None, distl_flags = None, threshold = None, xtal_target = None, negate_hits = False, trial_id = None, db_logging = False, progress_logging = False, sql_buffer_size = 1, db_host = None, db_name = None, db_table_name = None, db_experiment_tag = None, db_user = None, db_password = None, db_tags = None, rungroup_id = None, **kwds): """The mod_hitfind class constructor stores the parameters passed from the pyana configuration file in instance variables. All parameters, except @p address are optional, and hence need not be defined in pyana.cfg. @param address Full data source address of the DAQ device @param dispatch Function to call @param integration_dirname Directory portion of output integration file pathname @param integration_basename Filename prefix of output integration file pathname @param out_dirname Directory portion of output image pathname @param out_basename Filename prefix of output image pathname @param roi Region of interest for thresholding, on the form fast_low:fast_high,slow_low:slow_high @param threshold Minimum value in region of interest to pass """ super(mod_hitfind, self).__init__(address=address, **kwds) self.m_dispatch = cspad_tbx.getOptString(dispatch) self.m_integration_basename = cspad_tbx.getOptString(integration_basename) self.m_integration_dirname = cspad_tbx.getOptString(integration_dirname) self.m_out_basename = cspad_tbx.getOptString(out_basename) self.m_out_dirname = cspad_tbx.getOptString(out_dirname) self.m_distl_min_peaks = cspad_tbx.getOptInteger(distl_min_peaks) self.m_distl_flags = cspad_tbx.getOptStrings(distl_flags) self.m_threshold = cspad_tbx.getOptInteger(threshold) self.m_xtal_target = cspad_tbx.getOptString(xtal_target) self.m_negate_hits = cspad_tbx.getOptBool(negate_hits) self.m_trial_id = cspad_tbx.getOptInteger(trial_id) self.m_db_logging = cspad_tbx.getOptBool(db_logging) self.m_progress_logging = cspad_tbx.getOptBool(progress_logging) self.m_sql_buffer_size = cspad_tbx.getOptInteger(sql_buffer_size) self.m_db_host = cspad_tbx.getOptString(db_host) self.m_db_name = cspad_tbx.getOptString(db_name) self.m_db_table_name = cspad_tbx.getOptString(db_table_name) self.m_db_experiment_tag = cspad_tbx.getOptString(db_experiment_tag) self.m_db_user = cspad_tbx.getOptString(db_user) self.m_db_password = cspad_tbx.getOptString(db_password) self.m_db_tags = cspad_tbx.getOptString(db_tags) self.m_rungroup_id = cspad_tbx.getOptInteger(rungroup_id) # A ROI should not contain any ASIC boundaries, as these are # noisy. Hence circular ROI:s around the beam centre are probably # not such a grand idea. self.m_roi = cspad_tbx.getOptROI(roi) # Verify that dist_min_peaks is either "restrictive" or # "permissive", but not both. ^ is the logical xor operator if self.m_distl_min_peaks is not None: if (not (('permissive' in self.m_distl_flags) ^ ('restrictive' in self.m_distl_flags))): raise RuntimeError("""Sorry, with the distl_min_peaks option, distl_flags must be set to 'permissive' or 'restrictive'.""") if (self.m_roi is not None): raise RuntimeError("""Sorry, either specify region of interest (roi) or distl_min_peaks, but not both.""") if self.m_db_logging: self.buffered_sql_entries = [] assert self.m_sql_buffer_size >= 1 if self.m_progress_logging: self.buffered_progress_entries = [] assert self.m_sql_buffer_size >= 1 self.isoforms = {} if self.m_db_tags is None: self.m_db_tags = ""
def __init__(self, address, dispatch=None, integration_dirname=None, integration_basename=None, out_dirname=None, out_basename=None, roi=None, distl_min_peaks=None, distl_flags=None, threshold=None, xtal_target=None, negate_hits=False, trial_id=None, db_logging=False, progress_logging=False, sql_buffer_size=1, db_host=None, db_name=None, db_table_name=None, db_experiment_tag=None, db_user=None, db_password=None, db_tags=None, trial=None, rungroup_id=None, db_version='v1', **kwds): """The mod_hitfind class constructor stores the parameters passed from the pyana configuration file in instance variables. All parameters, except @p address are optional, and hence need not be defined in pyana.cfg. @param address Full data source address of the DAQ device @param dispatch Function to call @param integration_dirname Directory portion of output integration file pathname @param integration_basename Filename prefix of output integration file pathname @param out_dirname Directory portion of output image pathname @param out_basename Filename prefix of output image pathname @param roi Region of interest for thresholding, on the form fast_low:fast_high,slow_low:slow_high @param threshold Minimum value in region of interest to pass """ super(mod_hitfind, self).__init__(address=address, **kwds) self.m_dispatch = cspad_tbx.getOptString(dispatch) self.m_integration_basename = cspad_tbx.getOptString( integration_basename) self.m_integration_dirname = cspad_tbx.getOptString( integration_dirname) self.m_out_basename = cspad_tbx.getOptString(out_basename) self.m_out_dirname = cspad_tbx.getOptString(out_dirname) self.m_distl_min_peaks = cspad_tbx.getOptInteger(distl_min_peaks) self.m_distl_flags = cspad_tbx.getOptStrings(distl_flags) self.m_threshold = cspad_tbx.getOptInteger(threshold) self.m_xtal_target = cspad_tbx.getOptString(xtal_target) self.m_negate_hits = cspad_tbx.getOptBool(negate_hits) self.m_trial_id = cspad_tbx.getOptInteger(trial_id) self.m_db_logging = cspad_tbx.getOptBool(db_logging) self.m_progress_logging = cspad_tbx.getOptBool(progress_logging) self.m_sql_buffer_size = cspad_tbx.getOptInteger(sql_buffer_size) self.m_db_host = cspad_tbx.getOptString(db_host) self.m_db_name = cspad_tbx.getOptString(db_name) self.m_db_table_name = cspad_tbx.getOptString(db_table_name) self.m_db_experiment_tag = cspad_tbx.getOptString(db_experiment_tag) self.m_db_user = cspad_tbx.getOptString(db_user) self.m_db_password = cspad_tbx.getOptString(db_password) self.m_db_tags = cspad_tbx.getOptString(db_tags) self.m_trial = cspad_tbx.getOptInteger(trial) self.m_rungroup_id = cspad_tbx.getOptInteger(rungroup_id) self.m_db_version = cspad_tbx.getOptString(db_version) # A ROI should not contain any ASIC boundaries, as these are # noisy. Hence circular ROI:s around the beam centre are probably # not such a grand idea. self.m_roi = cspad_tbx.getOptROI(roi) # Verify that dist_min_peaks is either "restrictive" or # "permissive", but not both. ^ is the logical xor operator if self.m_distl_min_peaks is not None: if (not (('permissive' in self.m_distl_flags) ^ ('restrictive' in self.m_distl_flags))): raise RuntimeError("""Sorry, with the distl_min_peaks option, distl_flags must be set to 'permissive' or 'restrictive'.""") if (self.m_roi is not None): raise RuntimeError("""Sorry, either specify region of interest (roi) or distl_min_peaks, but not both.""") if self.m_db_logging: self.buffered_sql_entries = [] assert self.m_sql_buffer_size >= 1 if self.m_progress_logging: if self.m_db_version == 'v1': self.buffered_progress_entries = [] assert self.m_sql_buffer_size >= 1 self.isoforms = {} elif self.m_db_version == 'v2': from xfel.ui import db_phil_str from libtbx.phil import parse extra_phil = """ input { trial = None .type = int trial_id = None .type = int rungroup = None .type = int } """ self.db_params = parse(db_phil_str + extra_phil).extract() self.db_params.experiment_tag = self.m_db_experiment_tag self.db_params.db.host = self.m_db_host self.db_params.db.name = self.m_db_name self.db_params.db.user = self.m_db_user self.db_params.db.password = self.m_db_password self.db_params.input.trial = self.m_trial self.db_params.input.rungroup = self.m_rungroup_id if self.m_db_tags is None: self.m_db_tags = ""