def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).beginjob(evt, env)
self.set_up_hitfinder(env)
if self.m_db_logging:
from cxi_xdr_xes.cftbx.cspad_ana import db
self.logger.info("Connecting to db...")
dbobj = db.dbconnect(self.m_db_host, self.m_db_name,
self.m_db_user, self.m_db_password)
assert dbobj.open
self.logger.info("Connected.")
try:
self.trial = self.m_trial_id # TODO: beat the race condition and use db.get_next_trial_id if
# this is not set or is zero or less
db.create_tables(dbobj, self.m_db_table_name)
except Exception as e:
self.logger.info("Couldn't create root tables: %s" % (e))
dbobj.close()
def run(args):
master_phil = libtbx.phil.parse("""
db {
host = None
.type = str
name = None
.type = str
table_name = None
.type = str
user = None
.type = str
password = None
.type = str
}
""")
if (__name__ == "__main__") :
user_phil = []
for arg in args :
try :
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError, e :
raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry("Trial logging not supported for this installation. Conact the developers for access.")
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=params.db.password)
db.list_db_metadata(dbobj, params.db.table_name)
def commit_progress_entries(self):
if len(self.buffered_progress_entries) > 0:
print("Commiting %d entries to the db" %
len(self.buffered_progress_entries))
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name,
self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
for entry in self.buffered_progress_entries:
frame_sql, parameters, kwargs = entry['frame']
cursor.execute(frame_sql, parameters[0])
frame_id = cursor.lastrowid
_, _, kwargs = entry['observations']
kwargs['frames_id'] = [frame_id] * len(kwargs['frames_id'])
query = ("INSERT INTO `%s_observations` (" % self.m_db_experiment_tag) \
+ ", ".join(kwargs) + ") values (" \
+ ", ".join(["%s"] * len(kwargs)) + ")"
try:
parameters = list(zip(*list(kwargs.values())))
except TypeError:
parameters = [list(kwargs.values())]
cursor.executemany(query, parameters)
dbobj.commit()
cursor.close()
dbobj.close()
self.buffered_progress_entries = []
def run(args):
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry("Trial logging not supported for this installation. Contact the developers for access.")
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil)
params = phil.work.extract()
if params.db.host is None:
raise Usage("Please provide a host name")
if params.db.name is None:
raise Usage("Please provide a database name")
if params.db.user is None:
raise Usage("Please provide a user name")
if params.db.password is None:
import getpass
password = getpass.getpass()
else:
password = params.db.password
try:
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=password)
except Exception, e:
raise Sorry(e)
def run(args):
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry("Trial logging not supported for this installation. Contact the developers for access.")
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil)
params = phil.work.extract()
if params.db.host is None:
raise Usage("Please provide a host name")
if params.db.name is None:
raise Usage("Please provide a database name")
if params.db.user is None:
raise Usage("Please provide a user name")
if params.db.password is None:
import getpass
password = getpass.getpass()
else:
password = params.db.password
try:
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=password)
except Exception, e:
raise Sorry(e)
def commit_progress_entries(self):
if len(self.buffered_progress_entries) > 0:
print "Commiting %d entries to the db"%len(self.buffered_progress_entries)
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
for entry in self.buffered_progress_entries:
frame_sql, parameters, kwargs = entry['frame']
cursor.execute(frame_sql, parameters[0])
frame_id = cursor.lastrowid
_, _, kwargs = entry['observations']
kwargs['frames_id'] = [frame_id] * len(kwargs['frames_id'])
query = ("INSERT INTO `%s_observations` (" % self.m_db_experiment_tag) \
+ ", ".join(kwargs.keys()) + ") values (" \
+ ", ".join(["%s"] * len(kwargs.keys())) + ")"
try:
parameters = zip(*kwargs.values())
except TypeError:
parameters = [kwargs.values()]
cursor.executemany(query, parameters)
dbobj.commit()
cursor.close()
dbobj.close()
self.buffered_progress_entries = []
def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).beginjob(evt, env)
self.set_up_hitfinder()
if self.m_db_logging:
from cxi_xdr_xes.cftbx.cspad_ana import db
self.logger.info("Connecting to db...")
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
assert dbobj.open
self.logger.info("Connected.")
try:
self.trial = self.m_trial_id # TODO: beat the race condition and use db.get_next_trial_id if
# this is not set or is zero or less
db.create_tables(dbobj, self.m_db_table_name)
except Exception,e:
self.logger.info("Couldn't create root tables: %s"%(e))
dbobj.close()
def run(args):
master_phil = libtbx.phil.parse("""
db {
host = None
.type = str
name = None
.type = str
table_name = None
.type = str
user = None
.type = str
password = None
.type = str
}
""")
if (__name__ == "__main__") :
user_phil = []
for arg in args :
try :
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError as e :
raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry("Trial logging not supported for this installation. Conact the developers for access.")
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=params.db.password)
db.list_db_metadata(dbobj, params.db.table_name)
def commit_entries(self):
if len(self.buffered_sql_entries) > 0:
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
cmd = "INSERT INTO %s (trial,run,eventstamp,hitcount,distance,sifoil,wavelength,indexed,\
mosaic_block_rotation,mosaic_block_size,ewald_proximal_volume,spacegroup,\
cell_a,cell_b,cell_c,cell_alpha,cell_beta,cell_gamma,resolution,tags) VALUES "%(self.m_db_table_name)
comma = ""
for entry in self.buffered_sql_entries:
cmd += comma + "(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'%s',%s,%s,%s,%s,%s,%s,%s,'%s')"%entry
comma = ", "
cursor.execute(cmd)
dbobj.commit()
dbobj.close()
self.buffered_sql_entries = []
def commit_entries(self):
if len(self.buffered_sql_entries) > 0:
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
cmd = "INSERT INTO %s (trial,run,eventstamp,hitcount,distance,sifoil,wavelength,indexed,\
mosaic_block_rotation,mosaic_block_size,ewald_proximal_volume,spacegroup,\
cell_a,cell_b,cell_c,cell_alpha,cell_beta,cell_gamma,resolution,tags) VALUES "%(self.m_db_table_name)
comma = ""
for entry in self.buffered_sql_entries:
cmd += comma + "(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,'%s',%s,%s,%s,%s,%s,%s,%s,'%s')"%entry
comma = ", "
cursor.execute(cmd)
dbobj.commit()
dbobj.close()
self.buffered_sql_entries = []
def load_data (self):
ttop = time.time()
print "Loading data..."
assert (self.trial_id is not None)
import cxi_xdr_xes.cftbx.cspad_ana.db as cxidb
db=cxidb.dbconnect()
assert(db is not None and db.open)
# retrieve the run IDs in this trial
#t1 = time.time()
cursor = db.cursor()
#cursor.execute("SELECT DISTINCT(run) FROM %s WHERE trial = %s"%(cxidb.table_name,self.trial_id))
cmd = "SELECT DISTINCT(run) FROM %s WHERE trial = %s"
if self.params.run_num is not None:
extra = " AND run = %s"%self.params.run_num
elif self.params.run_min is not None and self.params.run_max is not None:
extra = " AND run >= %s AND run <= %s"%(self.params.run_min, self.params.run_max)
else:
extra = " ORDER BY run DESC LIMIT 5"
cursor.execute(cmd%(cxidb.table_name,self.trial_id) + extra)
#t2 = time.time()
#print "Runs queried in %.2fs" % (t2 - t1)
if(self.full_data_load):
self.runs = []
if(len(self.runs) > 5):
self.runs = self.runs[-5:]
new_data = False
for runId in cursor.fetchall():
if self.full_data_load:
run = Run(int(runId[0]))
self.runs.append(run)
else:
foundit=False
for runtest in self.runs:
if runtest.runId == int(runId[0]):
foundit = True
run = runtest
break
if not foundit:
print "New run: %s"%runId
run = Run(int(runId[0]))
self.runs.append(run)
#t1 = time.time()
#print "Loading data from run %s" % (run.runId)
if self.full_data_load or not hasattr(run, "latest_entry_id"):
print "Full load"
cursor.execute("SELECT id, eventstamp, hitcount, distance, sifoil, wavelength, indexed FROM %s \
WHERE trial = %s AND run = %s ORDER BY eventstamp"%(cxidb.table_name,self.trial_id,run.runId))
else:
print "Partial load"
cursor.execute("SELECT id, eventstamp, hitcount, distance, sifoil, wavelength, indexed FROM %s \
WHERE trial = %s AND run = %s AND id > %s ORDER BY eventstamp"%(cxidb.table_name,self.trial_id,run.runId,run.latest_entry_id ))
#t2 = time.time()
#print "Query ran in %.2fs" % (t2 - t1)
ids = flex.int()
for id, eventstamp, hitcount, distance, sifoil, wavelength, indexed in cursor.fetchall():
run.bragg_times.append(float(eventstamp))
run.braggs.append(int(hitcount))
ids.append(id)
run.distances.append(float(distance))
run.sifoils.append(float(sifoil))
run.wavelengths.append(float(wavelength))
run.indexed.append(bool(indexed))
if len(ids) > 0:
run.latest_entry_id = max(ids)
new_data = True
run.recalc_hits(self.params.average_window, self.params.hit_cutoff)
self.total_width = 0
for run in self.runs:
perm = flex.sort_permutation(run.hit_rates_times)
run.hit_rates_times = run.hit_rates_times.select(perm)
run.hit_rates = run.hit_rates.select(perm)
self.total_width += run.width()
self.cull_braggs()
#self.full_data_load = False #always do a full load
self.runs.sort(key=operator.attrgetter('runId'))
tbot = time.time()
print "Data loaded in %.2fs" % (tbot - ttop)
return new_data
def run(args):
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry("Trial logging not supported for this installation. Contact the developers for access.")
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil)
params = phil.work.extract()
if params.db.host is None:
raise Usage("Please provide a host name")
if params.db.name is None:
raise Usage("Please provide a database name")
if params.db.user is None:
raise Usage("Please provide a user name")
if params.db.password is None:
import getpass
password = getpass.getpass()
else:
password = params.db.password
try:
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=password)
except Exception as e:
raise Sorry(e)
from xfel.xpp.simulate import file_table
query = "https://pswww.slac.stanford.edu/ws-auth/dataexport/placed?exp_name=%s"%(params.experiment)
# set up extra run tags, if provided
if params.run_tags is not None:
extra1 = ", tags"
extra2 = ",'%s'"%params.run_tags
else:
extra1 = ""
extra2 = ""
while True:
# Get the set of known runs in the experiment database
cmd = "SELECT run from %s_runs"%params.experiment_tag
cursor = dbobj.cursor()
cursor.execute(cmd)
# Get the set of runs from SLAC's database
FT = file_table(params,query)
# Find the delta
known_runs = [int(entry[0]) for entry in cursor.fetchall()]
unknown_runs = [run for run in FT.rundict if run not in known_runs]
print "%d new runs"%len(unknown_runs)
# Enter any new runs into the experiment database
if len(unknown_runs) > 0:
cmd = "INSERT INTO %s_runs (run%s) VALUES "%(params.experiment_tag, extra1)
comma = ""
for run in unknown_runs:
cmd += comma + "(%d%s)"%(run, extra2)
comma = ", "
cursor = dbobj.cursor()
cursor.execute(cmd)
dbobj.commit()
time.sleep(10)
def run(args):
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry(
"Trial logging not supported for this installation. Conact the developers for access."
)
user_phil = []
# TODO: replace this stuff with iotbx.phil.process_command_line_with_files
# as soon as I can safely modify it
for arg in args:
#if (os.path.isdir(arg)) :
#user_phil.append(libtbx.phil.parse("""status_dir=\"%s\"""" % arg))
#elif (not "=" in arg) :
if (not "=" in arg):
try:
user_phil.append(
libtbx.phil.parse("""trial_id=%d""" % int(arg)))
except ValueError as e:
raise Sorry("Unrecognized argument '%s'" % arg)
else:
try:
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError as e:
raise Sorry("Unrecognized argument '%s' (error: %s)" %
(arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if (params.trial_id is None):
master_phil.show()
raise Usage(
"trial_id must be defined (either trial_id=XXX, or the integer " +
"ID alone).")
assert (params.hit_cutoff is not None) and (params.hit_cutoff > 0)
extra_cmd = ""
if params.run_start is not None:
extra_cmd += "AND run >= %d" % params.run_start
if params.run_end is not None:
extra_cmd += "AND run <= %d" % params.run_end
dbobj = db.dbconnect(host=params.db.host,
db=params.db.name,
username=params.db.user,
password=params.db.password)
cursor = dbobj.cursor()
cmd = "SELECT DISTINCT(run) FROM %s WHERE trial = %%s %s ORDER BY run" % (
params.db.table_name, extra_cmd)
cursor.execute(cmd, params.trial_id)
frames_total = 0
hits_total = 0
indexed_total = 0
for runId in cursor.fetchall():
run = int(runId[0])
cmd = "SELECT id, eventstamp, hitcount, distance, sifoil, wavelength, indexed FROM %s \
WHERE trial = %s AND run = %s"
if params.db.tags is not None:
for tag in params.db.tags.split(','):
cmd += """ AND tags LIKE "%%{0}%%" """.format(tag)
cursor.execute(cmd % (params.db.table_name, params.trial_id, run))
numframes = numhits = numindexed = 0
for id, eventstamp, hitcount, distance, sifoil, wavelength, indexed in cursor.fetchall(
):
numframes += 1
if hitcount >= params.hit_cutoff:
numhits += 1
if indexed:
numindexed += 1
if numhits == 0:
hitrate = 0
else:
hitrate = 100 * numhits / numframes
if numindexed == 0:
indexingrate = 0
else:
indexingrate = 100 * numindexed / numframes
print "Run: %3d, number of hits: %6d, number of frames: %6d, hitrate: %4.1f%%. Number indexed: %6d (%4.1f%%)" % (
run, numhits, numframes, hitrate, numindexed, indexingrate)
frames_total += numframes
hits_total += numhits
indexed_total += numindexed
if hits_total == 0:
hitrate = 0
else:
hitrate = 100 * hits_total / frames_total
if indexed_total == 0:
indexingrate = 0
else:
indexingrate = 100 * indexed_total / frames_total
print "Totals: frames: %d, hits: %d (%4.1f%%), indexed: %d (%4.1f%%)" % (
frames_total, hits_total, hitrate, indexed_total, indexingrate)
dbobj.close()
def run(args):
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry("Trial logging not supported for this installation. Contact the developers for access.")
phil = iotbx.phil.process_command_line(args=args, master_string=master_phil)
params = phil.work.extract()
if params.db.host is None:
raise Usage("Please provide a host name")
if params.db.name is None:
raise Usage("Please provide a database name")
if params.db.user is None:
raise Usage("Please provide a user name")
if params.db.password is None:
import getpass
password = getpass.getpass()
else:
password = params.db.password
while True:
print "Checking for new jobs to submit"
# need to get a new connection each iteration to prevent caching
try:
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=password)
except Exception, e:
raise Sorry(e)
# Get the set of known runs in the experiment database
cmd = "SELECT run from %s_runs"%params.experiment_tag
cursor = dbobj.cursor()
cursor.execute(cmd)
known_runs = [int(parse_entry(entry)[0]) for entry in cursor.fetchall()]
# Get the list of active trials
cmd = "SELECT trial_id, trial from %s_trials WHERE active = True"%params.experiment_tag
cursor = dbobj.cursor()
cursor.execute(cmd)
entries = [parse_entry(entry) for entry in cursor.fetchall()]
active_trial_ids = [int(entry[0]) for entry in entries]
active_trials = [int(entry[1]) for entry in entries]
for trial_id, trial in zip(active_trial_ids, active_trials):
# Get the active rungroups for this trial
cmd = "SELECT rungroups_id from %s_trial_rungroups WHERE trials_id = %d AND active = True"%(params.experiment_tag, trial_id)
cursor = dbobj.cursor()
cursor.execute(cmd)
active_rungroup_ids = [int(parse_entry(entry)[0]) for entry in cursor.fetchall()]
for rungroup_id in active_rungroup_ids:
# Get the list of runs for this rungroup
cmd = "SELECT startrun, endrun from %s_rungroups WHERE %s_rungroups.rungroup_id = %d"%(params.experiment_tag, params.experiment_tag, rungroup_id)
cursor = dbobj.cursor()
cursor.execute(cmd)
assert cursor.rowcount == 1
startrun, endrun = parse_entry(cursor.fetchall()[0])
cmd = "SELECT run_id, run from %s_runs WHERE run >= %d"%(params.experiment_tag, startrun)
if endrun is not None:
cmd += " AND run <= %d"%endrun
cursor = dbobj.cursor()
cursor.execute(cmd)
entries = [parse_entry(entry) for entry in cursor.fetchall()]
run_ids = [int(entry[0]) for entry in entries]
runs = [int(entry[1]) for entry in entries]
# Find the submitted runs for this trial/rungroup combination
cmd = "SELECT runs_id from %s_jobs WHERE trials_id = %d and rungroups_id = %d"%(params.experiment_tag, trial_id, rungroup_id)
cursor = dbobj.cursor()
cursor.execute(cmd)
submitted_run_ids = [int(parse_entry(entry)[0]) for entry in cursor.fetchall()]
# Submit new runs
for run_id, run in zip(run_ids, runs):
if run_id in submitted_run_ids:
continue
print "Submitting run %d into trial %d using rungroup %d"%(run_id, trial, rungroup_id)
config_path = write_hitfind_cfg(params, dbobj, trial_id, trial, rungroup_id)
if submit_job(params, dbobj, trial_id, trial, rungroup_id, run, config_path):
pass
cmd = "INSERT INTO %s_jobs (runs_id, trials_id, rungroups_id, status) VALUES (%d, %d, %d, '%s')"%(params.experiment_tag, run_id, trial_id, rungroup_id, "submitted")
cursor = dbobj.cursor()
cursor.execute(cmd)
dbobj.commit()
time.sleep(10)
except RuntimeError, e :
raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if (params.trial_id is None) :
master_phil.show()
raise Usage("trial_id must be defined (either trial_id=XXX, or the integer "+
"ID alone).")
assert (params.hit_cutoff is not None) and (params.hit_cutoff > 0)
extra_cmd = ""
if params.run_start is not None:
extra_cmd += "AND run >= %d" % params.run_start
if params.run_end is not None:
extra_cmd += "AND run <= %d" % params.run_end
dbobj = db.dbconnect(host=params.db.host, db=params.db.name, username=params.db.user, password=params.db.password)
cursor = dbobj.cursor()
cmd = "SELECT DISTINCT(run) FROM %s WHERE trial = %%s %s ORDER BY run"%(params.db.table_name, extra_cmd)
cursor.execute(cmd, params.trial_id)
frames_total = 0
hits_total = 0
indexed_total = 0
for runId in cursor.fetchall():
run = int(runId[0])
cmd = "SELECT id, eventstamp, hitcount, distance, sifoil, wavelength, indexed FROM %s \
WHERE trial = %s AND run = %s"
if params.db.tags is not None:
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights,outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(d,
integration_dirname = self.m_integration_dirname,
integration_basename = self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None
if indexed and self.m_progress_logging:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best["identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best["identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
if self.m_db_logging:
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead")
if (self.m_out_dirname is not None or
self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def application(params, loop=True):
from cxi_xdr_xes.cftbx.cspad_ana import db as cxidb
dbobj = cxidb.dbconnect(params.db.host, params.db.name, params.db.user,
params.db.password)
cursor = dbobj.cursor()
PM = progress_manager(params, cursor)
PM.setup_isoforms(cursor)
PM.setup_runtags(cursor)
isoforms = PM.isoforms
del dbobj
while 1:
dbobj = cxidb.dbconnect(params.db.host, params.db.name, params.db.user,
params.db.password)
cursor = dbobj.cursor()
results = {}
print "Looking for data..."
for tag in params.run_tags.split(','):
for isoform in isoforms:
M = PM.get_HKL(cursor, isoform=isoform, run_tags=tag)
cell = isoforms[isoform]['cell']
miller_set = mset(
anomalous_flag=False,
crystal_symmetry=symmetry(
unit_cell=cell,
space_group_symbol=isoforms[isoform]['lookup_symbol']),
indices=M)
miller_set.show_comprehensive_summary()
miller_set.setup_binner(d_min=params.resolution,
n_bins=params.n_bins)
given = miller_set.binner().counts_given()
ccomplete = miller_set.binner().counts_complete()
for i_bin in miller_set.binner().range_used():
sel = miller_set.binner().selection(i_bin)
self_sel = miller_set.select(sel)
d_max, d_min = self_sel.d_max_min()
compl = self_sel.completeness(d_max=d_max)
n_ref = sel.count(True)
if ccomplete[i_bin] == 0.:
multiplicity = 0.
else:
res_highest = d_min
multiplicity = given[i_bin] / ccomplete[i_bin]
d_range = miller_set.binner().bin_legend(
i_bin=i_bin, show_bin_number=False, show_counts=True)
fmt = "%3d: %-24s %4.2f %6d mult=%4.2f"
print fmt % (i_bin, d_range, compl, n_ref, multiplicity)
print
if len(tag) > 0:
key = "%s %s" % (tag, isoform)
else:
key = isoform
given_used = flex.int(given).select(
flex.size_t(miller_set.binner().range_used()))
ccomplete_used = flex.int(ccomplete).select(
flex.size_t(miller_set.binner().range_used()))
results[key] = dict(multiplicity=flex.sum(given_used) /
flex.sum(ccomplete_used),
completeness=miller_set.completeness(),
multiplicity_highest=multiplicity,
completeness_highest=compl,
resolution_highest=res_highest)
del dbobj
if not loop:
return results
time.sleep(10)
def load_data (self):
ttop = time.time()
print "Loading data..."
assert (self.trial_id is not None)
import cxi_xdr_xes.cftbx.cspad_ana.db as cxidb
db=cxidb.dbconnect()
assert(db is not None and db.open)
# retrieve the run IDs in this trial
#t1 = time.time()
cursor = db.cursor()
#cursor.execute("SELECT DISTINCT(run) FROM %s WHERE trial = %s"%(cxidb.table_name,self.trial_id))
cmd = "SELECT DISTINCT(run) FROM %s WHERE trial = %s"
if self.params.run_num is not None:
extra = " AND run = %s"%self.params.run_num
elif self.params.run_min is not None and self.params.run_max is not None:
extra = " AND run >= %s AND run <= %s"%(self.params.run_min, self.params.run_max)
else:
extra = " ORDER BY run DESC LIMIT 5"
cursor.execute(cmd%(cxidb.table_name,self.trial_id) + extra)
#t2 = time.time()
#print "Runs queried in %.2fs" % (t2 - t1)
if(self.full_data_load):
self.runs = []
if(len(self.runs) > 5):
self.runs = self.runs[-5:]
new_data = False
for runId in cursor.fetchall():
if self.full_data_load:
run = Run(int(runId[0]))
self.runs.append(run)
else:
foundit=False
for runtest in self.runs:
if runtest.runId == int(runId[0]):
foundit = True
run = runtest
break
if not foundit:
print "New run: %s"%runId
run = Run(int(runId[0]))
self.runs.append(run)
#t1 = time.time()
#print "Loading data from run %s" % (run.runId)
if self.full_data_load or not hasattr(run, "latest_entry_id"):
print "Full load"
cursor.execute("SELECT id, eventstamp, hitcount, distance, sifoil, wavelength, indexed FROM %s \
WHERE trial = %s AND run = %s ORDER BY eventstamp"%(cxidb.table_name,self.trial_id,run.runId))
else:
print "Partial load"
cursor.execute("SELECT id, eventstamp, hitcount, distance, sifoil, wavelength, indexed FROM %s \
WHERE trial = %s AND run = %s AND id > %s ORDER BY eventstamp"%(cxidb.table_name,self.trial_id,run.runId,run.latest_entry_id ))
#t2 = time.time()
#print "Query ran in %.2fs" % (t2 - t1)
ids = flex.int()
for id, eventstamp, hitcount, distance, sifoil, wavelength, indexed in cursor.fetchall():
run.bragg_times.append(float(eventstamp))
run.braggs.append(int(hitcount))
ids.append(id)
run.distances.append(float(distance))
run.sifoils.append(float(sifoil))
run.wavelengths.append(float(wavelength))
run.indexed.append(bool(indexed))
if len(ids) > 0:
run.latest_entry_id = max(ids)
new_data = True
run.recalc_hits(self.params.average_window, self.params.hit_cutoff)
self.total_width = 0
for run in self.runs:
perm = flex.sort_permutation(run.hit_rates_times)
run.hit_rates_times = run.hit_rates_times.select(perm)
run.hit_rates = run.hit_rates.select(perm)
self.total_width += run.width()
self.cull_braggs()
#self.full_data_load = False #always do a full load
self.runs.sort(key=operator.attrgetter('runId'))
tbot = time.time()
print "Data loaded in %.2fs" % (tbot - ttop)
return new_data
def application(params, loop = True):
from cxi_xdr_xes.cftbx.cspad_ana import db as cxidb
dbobj = cxidb.dbconnect(params.db.host, params.db.name, params.db.user, params.db.password)
cursor = dbobj.cursor()
PM = progress_manager(params, cursor)
PM.setup_isoforms(cursor)
PM.setup_runtags(cursor)
isoforms = PM.isoforms
del dbobj
while 1:
dbobj = cxidb.dbconnect(params.db.host, params.db.name, params.db.user, params.db.password)
cursor = dbobj.cursor()
results = {}
print "Looking for data..."
for tag in params.run_tags.split(','):
for isoform in isoforms:
M = PM.get_HKL(cursor,isoform=isoform,run_tags=tag)
cell = isoforms[isoform]['cell']
miller_set = mset(anomalous_flag = False, crystal_symmetry=symmetry(unit_cell=cell, space_group_symbol=isoforms[isoform]['lookup_symbol']), indices=M)
miller_set.show_comprehensive_summary()
miller_set.setup_binner(d_min=params.resolution, n_bins=params.n_bins)
given = miller_set.binner().counts_given()
ccomplete = miller_set.binner().counts_complete()
for i_bin in miller_set.binner().range_used():
sel = miller_set.binner().selection(i_bin)
self_sel = miller_set.select(sel)
d_max,d_min = self_sel.d_max_min()
compl = self_sel.completeness(d_max = d_max)
n_ref = sel.count(True)
if ccomplete[i_bin] == 0.:
multiplicity = 0.
else:
res_highest = d_min
multiplicity = given[i_bin]/ccomplete[i_bin]
d_range = miller_set.binner().bin_legend(
i_bin = i_bin, show_bin_number = False, show_counts = True)
fmt = "%3d: %-24s %4.2f %6d mult=%4.2f"
print fmt % (i_bin,d_range,compl,n_ref,
multiplicity)
print
if len(tag) > 0:
key = "%s %s"%(tag, isoform)
else:
key = isoform
given_used = flex.int(given).select(flex.size_t(miller_set.binner().range_used()))
ccomplete_used = flex.int(ccomplete).select(flex.size_t(miller_set.binner().range_used()))
results[key] = dict(
multiplicity = flex.sum(given_used)/flex.sum(ccomplete_used),
completeness = miller_set.completeness(),
multiplicity_highest = multiplicity,
completeness_highest = compl,
resolution_highest = res_highest
)
del dbobj
if not loop:
return results
time.sleep(10)
def run(args):
try:
from cxi_xdr_xes.cftbx.cspad_ana import db as db
except ImportError:
raise Sorry(
"Trial logging not supported for this installation. Contact the developers for access."
)
phil = iotbx.phil.process_command_line(args=args,
master_string=master_phil)
params = phil.work.extract()
if params.db.host is None:
raise Usage("Please provide a host name")
if params.db.name is None:
raise Usage("Please provide a database name")
if params.db.user is None:
raise Usage("Please provide a user name")
if params.db.password is None:
import getpass
password = getpass.getpass()
else:
password = params.db.password
while True:
print "Checking for new jobs to submit"
# need to get a new connection each iteration to prevent caching
try:
dbobj = db.dbconnect(host=params.db.host,
db=params.db.name,
username=params.db.user,
password=password)
except Exception as e:
raise Sorry(e)
# Get the set of known runs in the experiment database
cmd = "SELECT run from %s_runs" % params.experiment_tag
cursor = dbobj.cursor()
cursor.execute(cmd)
known_runs = [
int(parse_entry(entry)[0]) for entry in cursor.fetchall()
]
# Get the list of active trials
cmd = "SELECT trial_id, trial from %s_trials WHERE active = True" % params.experiment_tag
cursor = dbobj.cursor()
cursor.execute(cmd)
entries = [parse_entry(entry) for entry in cursor.fetchall()]
active_trial_ids = [int(entry[0]) for entry in entries]
active_trials = [int(entry[1]) for entry in entries]
for trial_id, trial in zip(active_trial_ids, active_trials):
# Get the active rungroups for this trial
cmd = "SELECT rungroups_id from %s_trial_rungroups WHERE trials_id = %d AND active = True" % (
params.experiment_tag, trial_id)
cursor = dbobj.cursor()
cursor.execute(cmd)
active_rungroup_ids = [
int(parse_entry(entry)[0]) for entry in cursor.fetchall()
]
for rungroup_id in active_rungroup_ids:
# Get the list of runs for this rungroup
cmd = "SELECT startrun, endrun from %s_rungroups WHERE %s_rungroups.rungroup_id = %d" % (
params.experiment_tag, params.experiment_tag, rungroup_id)
cursor = dbobj.cursor()
cursor.execute(cmd)
assert cursor.rowcount == 1
startrun, endrun = parse_entry(cursor.fetchall()[0])
cmd = "SELECT run_id, run from %s_runs WHERE run >= %d" % (
params.experiment_tag, startrun)
if endrun is not None:
cmd += " AND run <= %d" % endrun
cursor = dbobj.cursor()
cursor.execute(cmd)
entries = [parse_entry(entry) for entry in cursor.fetchall()]
run_ids = [int(entry[0]) for entry in entries]
runs = [int(entry[1]) for entry in entries]
# Find the submitted runs for this trial/rungroup combination
cmd = "SELECT runs_id from %s_jobs WHERE trials_id = %d and rungroups_id = %d" % (
params.experiment_tag, trial_id, rungroup_id)
cursor = dbobj.cursor()
cursor.execute(cmd)
submitted_run_ids = [
int(parse_entry(entry)[0]) for entry in cursor.fetchall()
]
# Submit new runs
for run_id, run in zip(run_ids, runs):
if run_id in submitted_run_ids:
continue
print "Submitting run %d into trial %d using rungroup %d" % (
run_id, trial, rungroup_id)
config_path = write_hitfind_cfg(params, dbobj, trial_id,
trial, rungroup_id)
if submit_job(params, dbobj, trial_id, trial, rungroup_id,
run, config_path):
pass
cmd = "INSERT INTO %s_jobs (runs_id, trials_id, rungroups_id, status) VALUES (%d, %d, %d, '%s')" % (
params.experiment_tag, run_id, trial_id, rungroup_id,
"submitted")
cursor = dbobj.cursor()
cursor.execute(cmd)
dbobj.commit()
time.sleep(10)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights, outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights >
self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((
peak_heights > self.m_threshold).__and__(
outvalue == 0)).count(True)
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
self.stats_logger.info("BRAGG %.3f %d" %
(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info(
"Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info(
"Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time,
number_of_accepted_peaks, distance, self.sifoil,
self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(
d,
integration_dirname=self.m_integration_dirname,
integration_basename=self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None and hasattr(info, 'spotfinder_results')
if self.m_progress_logging:
if self.m_db_version == 'v1':
if indexed:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best[
"identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host,
self.m_db_name,
self.m_db_user,
self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best[
"identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[
info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,
self.m_db_experiment_tag,
self.m_trial_id,
self.m_rungroup_id,
evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best[
"identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
elif self.m_db_version == 'v2':
key_low = 'cctbx.xfel.radial_average.two_theta_low'
key_high = 'cctbx.xfel.radial_average.two_theta_high'
tt_low = evt.get(key_low)
tt_high = evt.get(key_high)
from xfel.ui.db.dxtbx_db import log_frame
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
sfspots = evt.get('sfspots')
if sfspots is None:
if info is None or not isinstance(info, int):
n_spots = 0
else:
n_spots = info
else:
n_spots = sfspots
if indexed:
known_setting = info.horizons_phil.known_setting
indexed_setting = info.organizer.info[
'best_integration']['counter']
if known_setting is None or known_setting == indexed_setting:
from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
c = construct_reflection_table_and_experiment_list(
info.last_saved_best,
None,
pixel_size,
proceed_without_image=True)
c.assemble_experiments()
c.assemble_reflections()
log_frame(c.experiment_list, c.reflections,
self.db_params, evt.run(), n_spots,
self.timestamp, tt_low, tt_high)
else:
print(
"Not logging %s, wrong bravais setting (expecting %d, got %d)"
% (self.timestamp, known_setting,
indexed_setting))
else:
log_frame(None, None, self.db_params, evt.run(),
n_spots, self.timestamp, tt_low, tt_high)
if self.m_db_logging:
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get(
'ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get(
'ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get(
'ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
).parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time, n_spots,
distance, self.sifoil, self.wavelength, indexed,
mosaic_bloc_rotation, mosaic_block_size,
ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%" % detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(
horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%s" %
detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation, pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(
horizons_phil, imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead"
)
if (self.m_out_dirname is not None
or self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))