def getCurrentDetectorList(bl):
'''
returns available detector list for current beamline and newest run number
Detectors are cameras like the MPCCD
input: beamline as integer
output: tuple list of detector names
'''
return dbpy.read_detidlist(bl, getNewestRun(bl))
def getDetectorList(bl, run):
'''
returns available detector list for current beamline and run number
Detectors are cameras like the MPCCD
input: beamline and runnumber as integers
output: tuple list of detector names
'''
return dbpy.read_detidlist(bl, run)
def run(runid, bl=3, clen=50.0):
# Beamline specific constants
if bl == 2:
sensor_spec = "xfel_bl_2_tc_spec_1/energy"
sensor_shutter = "xfel_bl_2_shutter_1_open_valid/status"
elif bl == 3:
sensor_spec = "xfel_bl_3_tc_spec_1/energy"
sensor_shutter = "xfel_bl_3_shutter_1_open_valid/status"
else:
log_error("BadBeamline")
sys.exit(-1)
# Get Run info
try:
run_info = dbpy.read_runinfo(bl, runid)
except:
log_error("BadRunID")
sys.exit(-1)
high_tag = dbpy.read_hightagnumber(bl, runid)
start_tag = run_info['start_tagnumber']
end_tag = run_info['end_tagnumber']
tag_list = dbpy.read_taglist_byrun(bl, runid)
tag = tag_list[0]
print "Run %d: HighTag %d, Tags %d (inclusive) to %d (exclusive), thus %d images" % (
runid, high_tag, start_tag, end_tag, len(tag_list))
comment = dbpy.read_comment(bl, runid)
print "Comment: %s" % comment
print
# Find detectors
det_IDs = dbpy.read_detidlist(bl, runid)
print "Detector IDs: " + " ".join(det_IDs)
det_IDs = sorted([x for x in det_IDs if re.match("^MPCCD-8.*-[1-8]$", x)])
if len(det_IDs) != 8:
log_error("NoSupportedDetectorFound")
sys.exit(-1)
print "MPCCD Octal IDs to use: " + " ".join(det_IDs)
print
# Get shutter status and find dark images
try:
shutter = [
str2float(s)
for s in dbpy.read_syncdatalist(sensor_shutter, high_tag, tag_list)
]
except:
log_error("NoShutterStatus")
sys.exit(-1)
dark_tags = [
tag for tag, is_open in zip(tag_list, shutter) if is_open == 0
]
if bl == 2 and runid >= 32348: # and runid <= 33416:
# 2018 Feb: Unreliable shutter status. Should use PD and take darks only at the beginning of a run
print "The shutter status was unreliable for runs in 2018 Feb."
print "The number of tags with shutter closed:", len(dark_tags)
print "Since the above value is not reliable, we use X-ray PD values instead."
xray_pd = "xfel_bl_2_st_3_bm_1_pd/charge"
pd_values = [
str2float(s)
for s in dbpy.read_syncdatalist(xray_pd, high_tag, tag_list)
]
dark_tags = []
is_head = True
for tag, pd in zip(tag_list, pd_values):
if pd == None and is_head:
dark_tags.append(tag)
else:
is_head = False
print "Number of tags without X-ray:", len(
[1 for pd_val in pd_values if pd_val is None])
print "But we use only tags at the beginning of a run."
if len(dark_tags) == 0:
log_error("NoDarkImage")
sys.exit(-1)
print "Number of dark images to average: %d" % len(dark_tags)
print
# Setup buffer readers
try:
readers = [
stpy.StorageReader(det_id, bl, (runid, )) for det_id in det_IDs
]
except:
log_error("FailedOn_create_streader")
sys.exit(-1)
try:
buffers = [stpy.StorageBuffer(reader) for reader in readers]
except:
log_error("FailedOn_create_stbuf")
sys.exit(-1)
# Read first image to get detector info
det_infos = []
for reader, buf in zip(readers, buffers):
try:
reader.collect(buf, dark_tags[0])
except:
log_error("FailedOn_collect_data")
sys.exit(-1)
det_infos = [buf.read_det_info(0) for buf in buffers]
for i, det_info in enumerate(det_infos):
det_info['id'] = det_IDs[i]
# Collect pulse energies
config_photon_energy = 1000.0 * dbpy.read_config_photonenergy(bl, runid)
config_photon_energy_sensible = True
if config_photon_energy < 5000 or config_photon_energy > 14000:
print "WARNING: dbpy.read_config_photonenergy returned %f eV, which is absurd!" % config_photon_energy
print " Report this to SACLA DAQ team."
print " This is not problematic unless the inline spectrometer is also broken."
config_photon_energy_sensible = False
pulse_energies_in_keV = [
str2float(s) for s in dbpy.read_syncdatalist(sensor_spec, high_tag,
tuple(dark_tags))
]
pulse_energies = []
for tag, energy in zip(dark_tags, pulse_energies_in_keV):
if energy is not None and energy > 0:
pulse_energies.append(energy * 1000.0)
else:
print "WARNING: The wavelength from the inline spectrometer does not look sensible for tag %d." % tag
if config_photon_energy_sensible:
pulse_energies.append(config_photon_energy)
print " Used the accelerator config value instead."
else:
pulse_energies.append(7000.0)
print " The accelerator config value is also broken; assumed 7 keV as a last resort!"
print
mean_energy = np.mean(pulse_energies)
print "Mean photon energy: %f eV" % mean_energy
print "Configured photon energy: %f eV" % config_photon_energy
print
# Create geometry files
write_crystfel_geom("%d.geom" % runid, det_infos, mean_energy, clen)
write_cheetah_geom("%d-geom.h5" % runid, det_infos)
# Write metadata
write_metadata("%d.h5" % runid, det_infos, clen, comment)
# Create dark average
print
print "Calculating a dark average:"
num_added = 0
sum_buffer = np.zeros((YSIZE * NPANELS, XSIZE), dtype=np.float64)
gains = [det_info['mp_absgain'] for det_info in det_infos]
for j, tag_id in enumerate(dark_tags):
print "Processing tag %d (%2.1f%% done)" % (tag_id, 100.0 *
(j + 1) / len(dark_tags))
if (j % 5 == 0):
with open("status.txt", "w") as status:
status.write(
"Status: Total=%d,Processed=%d,Status=DarkAveraging\n" %
(len(dark_tags), j + 1))
num_added += add_image(sum_buffer, readers, buffers, gains, tag_id,
pulse_energies[j])
print "\nDone. Averaged %d frames." % num_added
if (num_added < 1):
return -1
sum_buffer /= num_added
ushort_max = np.iinfo(np.uint16).max
print " #neg (< 0) %d, #overflow (> %d) %d" % (np.sum(
sum_buffer < 0), ushort_max, np.sum(sum_buffer > ushort_max))
sum_buffer[sum_buffer < 0] = 0
sum_buffer[sum_buffer > ushort_max] = ushort_max
averaged = sum_buffer.astype(np.uint16)
# In the Phase 3 detector, some pixels average to negative values.
# Most are around -0.1 and all those below -1 are at panel edges that will be masked.
# So we don't have to worry about them.
f = h5py.File("%d-dark.h5" % runid, "w")
f.create_dataset("/data/data",
data=averaged,
compression="gzip",
shuffle=True)
# f.create_dataset("/data/raw", data=sum_buffer, compression="gzip", shuffle=True)
f.close()
print "Dark average was written to %s" % ("%d-dark.h5" % runid)
def run(runid, bl=3, clen=50.0, dry_run=False):
# Beamline specific constants
if bl == 2:
sensor_spec = "xfel_bl_2_tc_spec_1/energy"
sensor_shutter = "xfel_bl_2_shutter_1_open_valid/status"
elif bl == 3:
sensor_spec = "xfel_bl_3_tc_spec_1/energy"
sensor_shutter = "xfel_bl_3_shutter_1_open_valid/status"
else:
log_error("BadBeamline")
sys.exit(-1)
# Get Run info
try:
run_info = dbpy.read_runinfo(bl, runid)
except:
log_error("BadRunID")
sys.exit(-1)
high_tag = dbpy.read_hightagnumber(bl, runid)
start_tag = run_info['start_tagnumber']
end_tag = run_info['end_tagnumber']
tag_list = dbpy.read_taglist_byrun(bl, runid)
tag = tag_list[0]
print "Run %d: HighTag %d, Tags %d (inclusive) to %d (exclusive), thus %d images" % (runid, high_tag, start_tag, end_tag, len(tag_list))
comment = dbpy.read_comment(bl, runid)
print "Comment: %s" % comment
print
# Find detectors
det_IDs = dbpy.read_detidlist(bl, runid)
print "Detector IDs: " + " ".join(det_IDs)
det_IDs = sorted([x for x in det_IDs if re.match("^MPCCD-8.*-[1-8]$", x)])
if len(det_IDs) != 8:
log_error("NoSupportedDetectorFound")
sys.exit(-1)
print "MPCCD Octal IDs to use: " + " ".join(det_IDs)
print
# Get shutter status and find dark images
try:
shutter = [str2float(s) for s in dbpy.read_syncdatalist(sensor_shutter, high_tag, tag_list)]
except:
log_error("NoShutterStatus")
sys.exit(-1)
dark_tags = [tag for tag, is_open in zip(tag_list, shutter) if is_open == 0]
if bl == 2 and runid >= 32348: # and runid <= 33416:
# 2018 Feb: Unreliable shutter status. We should use BM1 PD and take darks only at the beginning of a run
print "The shutter status was unreliable for runs since 2018 Feb."
print "The number of tags with shutter closed:", len(dark_tags)
print "Since the above value is not reliable, we use X-ray PD values instead."
xray_pd = "xfel_bl_2_st_3_bm_1_pd/charge"
pd_values = [str2float(s) for s in dbpy.read_syncdatalist(xray_pd, high_tag, tag_list)]
dark_tags = []
is_head = True
for tag, pd in zip(tag_list, pd_values):
if math.isnan(pd) and is_head:
dark_tags.append(tag)
else:
is_head = False
print "Number of tags without X-ray:", len([1 for pd_val in pd_values if math.isnan(pd_val)])
print "But we use only tags at the beginning of a run."
if len(dark_tags) == 0:
log_error("NoDarkImage")
sys.exit(-1)
print "Number of dark images to average: %d" % len(dark_tags)
print
# Setup buffer readers
try:
readers = [stpy.StorageReader(det_id, bl, (runid,)) for det_id in det_IDs]
except:
log_error("FailedOn_create_streader")
sys.exit(-1)
try:
buffers = [stpy.StorageBuffer(reader) for reader in readers]
except:
log_error("FailedOn_create_stbuf")
sys.exit(-1)
# Read first image to get detector info
det_infos = []
for reader, buf in zip(readers, buffers):
try:
reader.collect(buf, dark_tags[0])
except:
log_error("FailedOn_collect_data")
sys.exit(-1)
det_infos = [buf.read_det_info(0) for buf in buffers]
for i, det_info in enumerate(det_infos):
det_info['id'] = det_IDs[i]
# Collect pulse energies
config_photon_energy = 1000.0 * dbpy.read_config_photonenergy(bl, runid)
config_photon_energy_sensible = True
if config_photon_energy < 5000 or config_photon_energy > 14000:
print "WARNING: dbpy.read_config_photonenergy returned %f eV, which is absurd!" % config_photon_energy
print " Report this to SACLA DAQ team."
print " This is not problematic unless the inline spectrometer is also broken."
config_photon_energy_sensible = False
pulse_energies_in_keV = [str2float(s) for s in dbpy.read_syncdatalist(sensor_spec, high_tag, tuple(dark_tags))]
pulse_energies = []
for tag, energy in zip(dark_tags, pulse_energies_in_keV):
if energy is not None and energy > 0:
pulse_energies.append(energy * 1000.0)
else:
print "WARNING: The wavelength from the inline spectrometer does not look sensible for tag %d." % tag
if config_photon_energy_sensible:
pulse_energies.append(config_photon_energy)
print " Used the accelerator config value instead."
else:
pulse_energies.append(7000.0)
print " The accelerator config value is also broken; assumed 7 keV as a last resort!"
print
mean_energy = np.mean(pulse_energies)
print "Mean photon energy: %f eV" % mean_energy
print "Configured photon energy: %f eV" % config_photon_energy
print
# Create geometry files
write_crystfel_geom("%d.geom" % runid, det_infos, mean_energy, clen)
write_cheetah_geom("%d-geom.h5" % runid, det_infos)
# Write metadata
write_metadata("%d.h5" % runid, det_infos, clen, comment)
if (dry_run): return
# Create dark average
print
print "Calculating a dark average:"
num_added = 0
sum_buffer = np.zeros((YSIZE * NPANELS, XSIZE), dtype=np.float64)
gains = [det_info['mp_absgain'] for det_info in det_infos]
for j, tag_id in enumerate(dark_tags):
print "Processing tag %d (%2.1f%% done)" % (tag_id, 100.0 * (j + 1) / len(dark_tags))
if (j % 5 == 0):
with open("status.txt", "w") as status:
status.write("Status: Total=%d,Processed=%d,Status=DarkAveraging\n" % (len(dark_tags), j + 1))
num_added += add_image(sum_buffer, readers, buffers, gains, tag_id, pulse_energies[j])
print "\nDone. Averaged %d frames." % num_added
if (num_added < 1):
return -1
sum_buffer /= num_added
ushort_max = np.iinfo(np.uint16).max
print " #neg (< 0) %d, #overflow (> %d) %d" % (np.sum(sum_buffer < 0), ushort_max, np.sum(sum_buffer > ushort_max))
sum_buffer[sum_buffer < 0] = 0
sum_buffer[sum_buffer > ushort_max] = ushort_max
averaged = sum_buffer.astype(np.uint16)
# In the Phase 3 detector, some pixels average to negative values.
# Most are around -0.1 and all those below -1 are at panel edges that will be masked.
# So we don't have to worry about them.
f = h5py.File("%d-dark.h5" % runid, "w")
f.create_dataset("/data/data", data=averaged, compression="gzip", shuffle=True)
# f.create_dataset("/data/raw", data=sum_buffer, compression="gzip", shuffle=True)
f.close()
print "Dark average was written to %s" % ("%d-dark.h5" % runid)
def getCurrentDetectorList(bl):
return dbpy.read_detidlist(bl, getNewestRun(bl))
def getDetectorList(bl, run):
return dbpy.read_detidlist(bl, run)