def getCurrentDetectorList(bl): ''' returns available detector list for current beamline and newest run number Detectors are cameras like the MPCCD input: beamline as integer output: tuple list of detector names ''' return dbpy.read_detidlist(bl, getNewestRun(bl))
def getDetectorList(bl, run): ''' returns available detector list for current beamline and run number Detectors are cameras like the MPCCD input: beamline and runnumber as integers output: tuple list of detector names ''' return dbpy.read_detidlist(bl, run)
def run(runid, bl=3, clen=50.0): # Beamline specific constants if bl == 2: sensor_spec = "xfel_bl_2_tc_spec_1/energy" sensor_shutter = "xfel_bl_2_shutter_1_open_valid/status" elif bl == 3: sensor_spec = "xfel_bl_3_tc_spec_1/energy" sensor_shutter = "xfel_bl_3_shutter_1_open_valid/status" else: log_error("BadBeamline") sys.exit(-1) # Get Run info try: run_info = dbpy.read_runinfo(bl, runid) except: log_error("BadRunID") sys.exit(-1) high_tag = dbpy.read_hightagnumber(bl, runid) start_tag = run_info['start_tagnumber'] end_tag = run_info['end_tagnumber'] tag_list = dbpy.read_taglist_byrun(bl, runid) tag = tag_list[0] print "Run %d: HighTag %d, Tags %d (inclusive) to %d (exclusive), thus %d images" % ( runid, high_tag, start_tag, end_tag, len(tag_list)) comment = dbpy.read_comment(bl, runid) print "Comment: %s" % comment print # Find detectors det_IDs = dbpy.read_detidlist(bl, runid) print "Detector IDs: " + " ".join(det_IDs) det_IDs = sorted([x for x in det_IDs if re.match("^MPCCD-8.*-[1-8]$", x)]) if len(det_IDs) != 8: log_error("NoSupportedDetectorFound") sys.exit(-1) print "MPCCD Octal IDs to use: " + " ".join(det_IDs) print # Get shutter status and find dark images try: shutter = [ str2float(s) for s in dbpy.read_syncdatalist(sensor_shutter, high_tag, tag_list) ] except: log_error("NoShutterStatus") sys.exit(-1) dark_tags = [ tag for tag, is_open in zip(tag_list, shutter) if is_open == 0 ] if bl == 2 and runid >= 32348: # and runid <= 33416: # 2018 Feb: Unreliable shutter status. Should use PD and take darks only at the beginning of a run print "The shutter status was unreliable for runs in 2018 Feb." print "The number of tags with shutter closed:", len(dark_tags) print "Since the above value is not reliable, we use X-ray PD values instead." xray_pd = "xfel_bl_2_st_3_bm_1_pd/charge" pd_values = [ str2float(s) for s in dbpy.read_syncdatalist(xray_pd, high_tag, tag_list) ] dark_tags = [] is_head = True for tag, pd in zip(tag_list, pd_values): if pd == None and is_head: dark_tags.append(tag) else: is_head = False print "Number of tags without X-ray:", len( [1 for pd_val in pd_values if pd_val is None]) print "But we use only tags at the beginning of a run." if len(dark_tags) == 0: log_error("NoDarkImage") sys.exit(-1) print "Number of dark images to average: %d" % len(dark_tags) print # Setup buffer readers try: readers = [ stpy.StorageReader(det_id, bl, (runid, )) for det_id in det_IDs ] except: log_error("FailedOn_create_streader") sys.exit(-1) try: buffers = [stpy.StorageBuffer(reader) for reader in readers] except: log_error("FailedOn_create_stbuf") sys.exit(-1) # Read first image to get detector info det_infos = [] for reader, buf in zip(readers, buffers): try: reader.collect(buf, dark_tags[0]) except: log_error("FailedOn_collect_data") sys.exit(-1) det_infos = [buf.read_det_info(0) for buf in buffers] for i, det_info in enumerate(det_infos): det_info['id'] = det_IDs[i] # Collect pulse energies config_photon_energy = 1000.0 * dbpy.read_config_photonenergy(bl, runid) config_photon_energy_sensible = True if config_photon_energy < 5000 or config_photon_energy > 14000: print "WARNING: dbpy.read_config_photonenergy returned %f eV, which is absurd!" % config_photon_energy print " Report this to SACLA DAQ team." print " This is not problematic unless the inline spectrometer is also broken." config_photon_energy_sensible = False pulse_energies_in_keV = [ str2float(s) for s in dbpy.read_syncdatalist(sensor_spec, high_tag, tuple(dark_tags)) ] pulse_energies = [] for tag, energy in zip(dark_tags, pulse_energies_in_keV): if energy is not None and energy > 0: pulse_energies.append(energy * 1000.0) else: print "WARNING: The wavelength from the inline spectrometer does not look sensible for tag %d." % tag if config_photon_energy_sensible: pulse_energies.append(config_photon_energy) print " Used the accelerator config value instead." else: pulse_energies.append(7000.0) print " The accelerator config value is also broken; assumed 7 keV as a last resort!" print mean_energy = np.mean(pulse_energies) print "Mean photon energy: %f eV" % mean_energy print "Configured photon energy: %f eV" % config_photon_energy print # Create geometry files write_crystfel_geom("%d.geom" % runid, det_infos, mean_energy, clen) write_cheetah_geom("%d-geom.h5" % runid, det_infos) # Write metadata write_metadata("%d.h5" % runid, det_infos, clen, comment) # Create dark average print print "Calculating a dark average:" num_added = 0 sum_buffer = np.zeros((YSIZE * NPANELS, XSIZE), dtype=np.float64) gains = [det_info['mp_absgain'] for det_info in det_infos] for j, tag_id in enumerate(dark_tags): print "Processing tag %d (%2.1f%% done)" % (tag_id, 100.0 * (j + 1) / len(dark_tags)) if (j % 5 == 0): with open("status.txt", "w") as status: status.write( "Status: Total=%d,Processed=%d,Status=DarkAveraging\n" % (len(dark_tags), j + 1)) num_added += add_image(sum_buffer, readers, buffers, gains, tag_id, pulse_energies[j]) print "\nDone. Averaged %d frames." % num_added if (num_added < 1): return -1 sum_buffer /= num_added ushort_max = np.iinfo(np.uint16).max print " #neg (< 0) %d, #overflow (> %d) %d" % (np.sum( sum_buffer < 0), ushort_max, np.sum(sum_buffer > ushort_max)) sum_buffer[sum_buffer < 0] = 0 sum_buffer[sum_buffer > ushort_max] = ushort_max averaged = sum_buffer.astype(np.uint16) # In the Phase 3 detector, some pixels average to negative values. # Most are around -0.1 and all those below -1 are at panel edges that will be masked. # So we don't have to worry about them. f = h5py.File("%d-dark.h5" % runid, "w") f.create_dataset("/data/data", data=averaged, compression="gzip", shuffle=True) # f.create_dataset("/data/raw", data=sum_buffer, compression="gzip", shuffle=True) f.close() print "Dark average was written to %s" % ("%d-dark.h5" % runid)
def run(runid, bl=3, clen=50.0, dry_run=False): # Beamline specific constants if bl == 2: sensor_spec = "xfel_bl_2_tc_spec_1/energy" sensor_shutter = "xfel_bl_2_shutter_1_open_valid/status" elif bl == 3: sensor_spec = "xfel_bl_3_tc_spec_1/energy" sensor_shutter = "xfel_bl_3_shutter_1_open_valid/status" else: log_error("BadBeamline") sys.exit(-1) # Get Run info try: run_info = dbpy.read_runinfo(bl, runid) except: log_error("BadRunID") sys.exit(-1) high_tag = dbpy.read_hightagnumber(bl, runid) start_tag = run_info['start_tagnumber'] end_tag = run_info['end_tagnumber'] tag_list = dbpy.read_taglist_byrun(bl, runid) tag = tag_list[0] print "Run %d: HighTag %d, Tags %d (inclusive) to %d (exclusive), thus %d images" % (runid, high_tag, start_tag, end_tag, len(tag_list)) comment = dbpy.read_comment(bl, runid) print "Comment: %s" % comment print # Find detectors det_IDs = dbpy.read_detidlist(bl, runid) print "Detector IDs: " + " ".join(det_IDs) det_IDs = sorted([x for x in det_IDs if re.match("^MPCCD-8.*-[1-8]$", x)]) if len(det_IDs) != 8: log_error("NoSupportedDetectorFound") sys.exit(-1) print "MPCCD Octal IDs to use: " + " ".join(det_IDs) print # Get shutter status and find dark images try: shutter = [str2float(s) for s in dbpy.read_syncdatalist(sensor_shutter, high_tag, tag_list)] except: log_error("NoShutterStatus") sys.exit(-1) dark_tags = [tag for tag, is_open in zip(tag_list, shutter) if is_open == 0] if bl == 2 and runid >= 32348: # and runid <= 33416: # 2018 Feb: Unreliable shutter status. We should use BM1 PD and take darks only at the beginning of a run print "The shutter status was unreliable for runs since 2018 Feb." print "The number of tags with shutter closed:", len(dark_tags) print "Since the above value is not reliable, we use X-ray PD values instead." xray_pd = "xfel_bl_2_st_3_bm_1_pd/charge" pd_values = [str2float(s) for s in dbpy.read_syncdatalist(xray_pd, high_tag, tag_list)] dark_tags = [] is_head = True for tag, pd in zip(tag_list, pd_values): if math.isnan(pd) and is_head: dark_tags.append(tag) else: is_head = False print "Number of tags without X-ray:", len([1 for pd_val in pd_values if math.isnan(pd_val)]) print "But we use only tags at the beginning of a run." if len(dark_tags) == 0: log_error("NoDarkImage") sys.exit(-1) print "Number of dark images to average: %d" % len(dark_tags) print # Setup buffer readers try: readers = [stpy.StorageReader(det_id, bl, (runid,)) for det_id in det_IDs] except: log_error("FailedOn_create_streader") sys.exit(-1) try: buffers = [stpy.StorageBuffer(reader) for reader in readers] except: log_error("FailedOn_create_stbuf") sys.exit(-1) # Read first image to get detector info det_infos = [] for reader, buf in zip(readers, buffers): try: reader.collect(buf, dark_tags[0]) except: log_error("FailedOn_collect_data") sys.exit(-1) det_infos = [buf.read_det_info(0) for buf in buffers] for i, det_info in enumerate(det_infos): det_info['id'] = det_IDs[i] # Collect pulse energies config_photon_energy = 1000.0 * dbpy.read_config_photonenergy(bl, runid) config_photon_energy_sensible = True if config_photon_energy < 5000 or config_photon_energy > 14000: print "WARNING: dbpy.read_config_photonenergy returned %f eV, which is absurd!" % config_photon_energy print " Report this to SACLA DAQ team." print " This is not problematic unless the inline spectrometer is also broken." config_photon_energy_sensible = False pulse_energies_in_keV = [str2float(s) for s in dbpy.read_syncdatalist(sensor_spec, high_tag, tuple(dark_tags))] pulse_energies = [] for tag, energy in zip(dark_tags, pulse_energies_in_keV): if energy is not None and energy > 0: pulse_energies.append(energy * 1000.0) else: print "WARNING: The wavelength from the inline spectrometer does not look sensible for tag %d." % tag if config_photon_energy_sensible: pulse_energies.append(config_photon_energy) print " Used the accelerator config value instead." else: pulse_energies.append(7000.0) print " The accelerator config value is also broken; assumed 7 keV as a last resort!" print mean_energy = np.mean(pulse_energies) print "Mean photon energy: %f eV" % mean_energy print "Configured photon energy: %f eV" % config_photon_energy print # Create geometry files write_crystfel_geom("%d.geom" % runid, det_infos, mean_energy, clen) write_cheetah_geom("%d-geom.h5" % runid, det_infos) # Write metadata write_metadata("%d.h5" % runid, det_infos, clen, comment) if (dry_run): return # Create dark average print print "Calculating a dark average:" num_added = 0 sum_buffer = np.zeros((YSIZE * NPANELS, XSIZE), dtype=np.float64) gains = [det_info['mp_absgain'] for det_info in det_infos] for j, tag_id in enumerate(dark_tags): print "Processing tag %d (%2.1f%% done)" % (tag_id, 100.0 * (j + 1) / len(dark_tags)) if (j % 5 == 0): with open("status.txt", "w") as status: status.write("Status: Total=%d,Processed=%d,Status=DarkAveraging\n" % (len(dark_tags), j + 1)) num_added += add_image(sum_buffer, readers, buffers, gains, tag_id, pulse_energies[j]) print "\nDone. Averaged %d frames." % num_added if (num_added < 1): return -1 sum_buffer /= num_added ushort_max = np.iinfo(np.uint16).max print " #neg (< 0) %d, #overflow (> %d) %d" % (np.sum(sum_buffer < 0), ushort_max, np.sum(sum_buffer > ushort_max)) sum_buffer[sum_buffer < 0] = 0 sum_buffer[sum_buffer > ushort_max] = ushort_max averaged = sum_buffer.astype(np.uint16) # In the Phase 3 detector, some pixels average to negative values. # Most are around -0.1 and all those below -1 are at panel edges that will be masked. # So we don't have to worry about them. f = h5py.File("%d-dark.h5" % runid, "w") f.create_dataset("/data/data", data=averaged, compression="gzip", shuffle=True) # f.create_dataset("/data/raw", data=sum_buffer, compression="gzip", shuffle=True) f.close() print "Dark average was written to %s" % ("%d-dark.h5" % runid)
def getCurrentDetectorList(bl): return dbpy.read_detidlist(bl, getNewestRun(bl))
def getDetectorList(bl, run): return dbpy.read_detidlist(bl, run)