def write_chunk(args, chunk, frames, vds_fname, out):
import sys
import glob
# The following line works on Maxwell
sys.path.append('/home/ayyerkar/.local/dragonfly/utils/py_src')
import writeemc
import os
post_tag = ['_lowq.h5', '_medq.h5', '_allq.h5'][2]
# Module order matches geometry file
modules = [[3, 4, 8, 15], [2, 3, 4, 5, 9, 8, 15, 14], np.arange(16)][2]
subset = [128, 256, 0][2]
num_pix = [4 * 128 * 128, 4 * 256 * 256, 4 * 512 * 512][2]
shift = args.threshold - 0.5
emcfile = os.path.join(out, "r%04d_%06d" % (args.run, chunk) + post_tag)
emc = writeemc.EMCWriter(emcfile, num_pix)
photon_ADU = 45.
print('Calibrating virtual data set for run %d' % args.run)
with agipd_vds.AGIPD_VDS_Calibrator(vds_fname,
calib_run=args.calib_run,
verbose=int(args.verbose)) as c:
print('Calibrating %s frames from run %d' % (len(frames), args.run))
frame = c.get_frame(frames, calibrate=True, assemble=False)
if (frame is None):
print("Chunk %04d is empty" % chunk)
os.system('rm %s' % (emcfile))
return
print('Converting %d frames from run %d' % (len(frames), args.run))
nframes = frame.shape[0]
for i in range(nframes):
emc.write_frame(
np.round(frame[i][modules, -subset:] / photon_ADU -
shift).ravel().astype('i4'))
sys.stderr.write('\r%d/%d' % (i + 1, nframes))
sys.stderr.write('\n')
emc.finish_write()
with h5py.File(emcfile, 'a') as f:
f['id/cells'] = c.cell_ids
f['id/pulses'] = c.pulse_ids
f['id/trains'] = c.train_ids
print("Chunk %04d is finished" % chunk)
os.system('chmod ao+rw %s' % (emcfile))
print('Reading from file:', allfile)
allemc = reademc.EMCReader(allfile, det)
# Load cells and scores
with sparse.Run(allfile) as r:
cells = r.cellIds
score = r.litpixel
goodcells = np.ones(len(cells), dtype=np.bool)
goodcells[cells == 0] = False
if args.run <= 577:
goodcells[cells >= 142] = False
# Save hits only
hitfile = os.path.join(args.output, "r%04d.h5" % (args.run))
print("Saving to new emc file: ", hitfile)
hitemc = writeemc.EMCWriter(hitfile, det.raw_mask.shape)
for i in range(allemc.num_frames):
hit = score[i] > args.threshold
if hit and goodcells[i]:
hitemc.write_frame(allemc.get_frame(i, raw=True))
sys.stderr.write('\r%d/%d' % (i + 1, allemc.num_frames))
sys.stderr.write('\n')
hitemc.finish_write()
print('copying ids/scores from: %s' % allfile)
selection = (score > args.threshold) & (goodcells)
with h5py.File(allfile, "r") as af:
with h5py.File(hitfile, "a") as hf:
if "id" in hf:
del hf["id"]
num_pix = [4 * 128 * 128, 4 * 256 * 256, 4 * 512 * 512][args.res]
else:
print('"res" parameter can only have values 0, 1')
sys.exit(1)
allqdet = detector.Detector(
'/gpfs/exfel/exp/SPB/201901/p002316/scratch/benedikt/aux/det_allq.h5')
allqfile = os.path.join(args.path, 'r%04d.h5' % args.run)
print('Cropping the following files:', allqfile)
allq_emc = reademc.EMCReader(allqfile, allqdet)
det = detector.Detector(
'/gpfs/exfel/exp/SPB/201901/p002316/scratch/benedikt/aux/det' + post_tag)
emcfile = os.path.join(args.path, post_tag[1:-3] + "/r%04d.h5" % (args.run))
print("Saving to new emc file: ", emcfile)
emc = writeemc.EMCWriter(emcfile, num_pix)
for i in range(allq_emc.num_frames):
frame = allq_emc.get_frame(i, raw=True).reshape((16, 512, 128))
emc.write_frame(frame[modules, -subset:].ravel().astype('i4'))
sys.stderr.write('\r%d/%d' % (i + 1, allq_emc.num_frames))
sys.stderr.write('\n')
emc.finish_write()
print('copying ids/scores from: %s' % allqfile)
with h5py.File(allqfile, "r") as af:
with h5py.File(emcfile, "a") as cf:
if "id" in cf:
del cf["id"]
if "scores" in cf:
del cf["scores"]
import h5py
import argparse
sys.path.append('/home/ayyerkar/.local/dragonfly/utils/py_src')
import writeemc
import os
parser = argparse.ArgumentParser(description='Convert hits file to sparse EMC H5 file')
parser.add_argument('-r', '--run', type=int, help='Run number')
parser.add_argument('-p', '--path', type=str, help='Path to hits files',default = '/gpfs/exfel/exp/SQS/201901/p002146/scratch/hits')
parser.add_argument('-o', '--out_folder', help='Path to output folder',default='/gpfs/exfel/exp/SQS/201901/p002146/scratch/emc')
parser.add_argument('-t', '--adu_threshold', help='ADU threshold', type=float, default=200)
args = parser.parse_args()
hitfiles = os.path.join(args.path + "/r%04d_hits.h5" %(args.run))
emcfile = os.path.join(args.out_folder, 'r%04d_emc.h5'%(args.run))
emc = writeemc.EMCWriter(emcfile, 1024*1024)
photon_ADU = args.adu_threshold
with h5py.File(hitfiles,'r') as fptr:
nframes = fptr['hits'].shape[0]
for i in range(nframes):
emc.write_frame(fptr['hits'][i].ravel().astype('i4'))
#emc.write_frame(np.round(fptr['hits'][i]/photon_ADU).ravel().astype('i4'))
sys.stderr.write('\r%d/%d'%(i+1, nframes))
emc.finish_write()
os.system('chmod ao+rw %s' % (emcfile))
parser.add_argument('run', type=int, help='Run number')
parser.add_argument('-p', '--path', type=str, help='Path to chunked sparse files',
default='/gpfs/exfel/exp/SPB/201802/p002145/scratch/sparse/chunks')
parser.add_argument('-o', '--out_folder', help='Path to output folder',
default='/gpfs/exfel/exp/SPB/201802/p002145/scratch/sparse/')
parser.add_argument('-d', '--delete', action='store_false', default=True,
help='Use this option for not deleting the chunks afterwards')
args = parser.parse_args()
# Merge sparse data
det = detector.Detector('/gpfs/exfel/exp/SPB/201802/p002160/scratch/ayyerkar/det/det_2160_allq2.h5')
chunked_flist = sorted(glob.glob(os.path.join(args.path, 'r%04d' %args.run, '*r%04d_*'%args.run)))
print('Merging %d chunked files' %len(chunked_flist))
chunked_emc = reademc.EMCReader(chunked_flist, det)
emcfile = os.path.join(args.out_folder, "r%04d.h5" %(args.run))
combined_emc = writeemc.EMCWriter(emcfile, det.raw_mask.shape)
for i in range(chunked_emc.num_frames):
combined_emc.write_frame(chunked_emc.get_frame(i, raw=True))
sys.stderr.write('\r%d/%d'%(i+1, chunked_emc.num_frames))
sys.stderr.write('\n')
nframes = chunked_emc.num_frames
combined_emc.finish_write()
print("Merged %d frames into %s" %(nframes, emcfile))
# Merge train, cell, pulse id
train_ids, cell_ids, pulse_ids = [], [], []
for filename in sorted(chunked_flist):
with h5py.File(filename, 'r') as f:
train_ids = np.hstack([train_ids, f['id/trains'][:]])
cell_ids = np.hstack([cell_ids, f['id/cells'][:]])
pulse_ids = np.hstack([pulse_ids, f['id/pulses'][:]])