def get_data(datdir, prefd, sufd, nf1, nf2):
# read Maxipix, Eiger edf images ####
t0 = time.time()
swrite = stdout.write
sflush = stdout.flush
print("start reading the files")
# creating filenames
filenames = nfiles.filename(datdir + prefd, sufd, nf1, nf2)
lfilenames = len(filenames)
# reading first image to get dimenstions of the matrix
headers = EdfMethods.headeredf(filenames[0])
dim1 = np.intc(headers['Dim_1'])
dim2 = np.intc(headers['Dim_2'])
nx = dim2
ny = dim1
data = np.zeros((lfilenames, nx, ny), np.uint16)
i = 0
for mfile in filenames:
data[i, :, :] = EdfMethods.loadedf(mfile)
i += 1
swrite(4 * '\x08')
swrite(str(int(i * 100. / lfilenames)) + '%')
sflush()
print("\n")
print("Reading time %3.3f sec" % (time.time() - t0))
return data
def read_det_mask(det_mask, detector):
""" read detector mask file
:param det_mask: filename that contains detector mask
:return: dmask, 2D array of beamstop mask, 1 is masked values
"""
if det_mask != 'none':
try:
dmask = np.abs(EdfMethods.loadedf(det_mask)) #reads edf and npy
dmask[dmask > 1] = 1
except:
print("Cannot read detector mask %s, skip" % det_mask)
else:
if detector == 'Andor':
dshape = (1024, 1024)
elif detector == 'maxipix':
dshape = (516, 516)
elif detector == 'eiger500K':
dshape = (1024, 514)
elif detector == 'eiger4m':
dshape = (2162, 2068)
else:
print("Needs detector mask file")
sys.exit()
dmask = np.zeros(dshape, dtype='uint8')
return dmask
def mread_eiger(mfilename, mdataout, ind_g, mNp, nx):
for mfile in mfilename:
matr = EdfMethods.loadedf(mfile)
msumpix, mpix = eigerpix(matr[ind_g], mNp, nx)
mpix = mpix[:msumpix]
mdataout.put([mpix, msumpix, matr])
mdataout.close()
def mread_ccd(mfilename, mdataout, darkimg, lth, bADU, tADU, mNp, aduph, nx,
ny):
for mfile in mfilename:
matr = np.asfarray(EdfMethods.loadedf(mfile), dtype=np.float32)
# try:
# matr[ind] = 0
# except:
# pass
msumpix, mpix, tmp = dropimgood(matr, darkimg, lth, bADU, tADU, mNp,
aduph, nx, ny) # dropletize CCD frames
mpix = mpix[:msumpix]
mdataout.put([mpix, msumpix, tmp])
mdataout.close()
def read_beamstop_mask(beamstop_mask):
""" read beamstop mask file
:param beamstop_mask: filename that contains beamstop mask
:return: bmask, 2D array of beamstop mask, 1 is masked values
"""
if beamstop_mask != 'none':
try:
bmask = np.abs(
EdfMethods.loadedf(beamstop_mask)) #reads edf and npy
bmask[bmask > 1] = 1
except:
print("Cannot read beamstop mask %s, skip" % beamstop_mask)
else:
bmask = 0
return bmask
def read_qmask(qmask_file, mask, number_q):
""" read qmask file
:param qmask_file: filename that contains qmask
:param mask: detector mask 2D array
:param number_q: number of qs in q mask
:return: qqmask, 2D array of q mask each reqion is 1,2,3...
"""
if qmask_file != 'none':
try:
qqmask = EdfMethods.loadedf(qmask_file)
qqmask[mask > 0] = 0
qqmask[qqmask > number_q] = 0
except:
print("Cannot read qmask %s, skip" % qmask_file)
qqmask = mask * 0
qqmask[mask < 1] = 1
else:
qqmask = mask * 0
qqmask[mask < 1] = 1
return qqmask
def main():
################# READING CONFIG FILE ##########################################################################
try:
config.read(sys.argv[1])
except:
print("Cannot read " + sys.argv[1] + " input file")
exit()
#### Sample description ######################################################################################
sname = config["sample_description"]["name"]
#### Data location ######################################################################################
datdir = config["data_location"]["data_dir"]
sample_dir = config["data_location"]["sample_dir"]
prefd = config["data_location"]["data_prefix"]
sufd = config["data_location"]["data_sufix"]
nf1 = int(config["data_location"]["first_file"])
nf2 = int(config["data_location"]["last_file"])
darkdir = config["data_location"]["dark_dir"]
df1 = int(config["data_location"]["first_dark"])
df2 = int(config["data_location"]["last_dark"])
savdir = config["data_location"]["result_dir"]
#### Experimental setup ######################################################################################
geometry = config["exp_setup"]["geometry"]
cx = float(config["exp_setup"]["dbx"])
cy = float(config["exp_setup"]["dby"])
dt = float(config["exp_setup"]["lagtime"])
lambdaw = float(config["exp_setup"]["wavelength"])
distance = float(config["exp_setup"]["detector_distance"])
first_q = float(config["exp_setup"]["firstq"])
width_q = float(config["exp_setup"]["widthq"])
step_q = float(config["exp_setup"]["stepq"])
number_q = int(config["exp_setup"]["numberq"])
beamstop_mask = config["exp_setup"]["beamstop_mask"]
#### Correlator info ######################################################################################
correlator = config["correlator"]["method"]
lth = int(config["correlator"]["low_threshold"])
bADU = int(config["correlator"]["bottom_ADU"])
tADU = int(config["correlator"]["top_ADU"])
mNp = float(config["correlator"]["max_number"])
aduph = int(config["correlator"]["photon_ADU"])
ttcf_par = int(config["correlator"]["ttcf"])
#### Detector description ######################################################################################
detector = config["detector"]["det_name"]
pix_size = float(config["detector"]["pixels"])
mask_file = config["detector"]["mask"]
flatfield_file = config["detector"]["flatfield"]
###################################################################################################################
import pylab as plt
print("Making qs")
try:
data = readdata.readnpz(savdir + sname + "_2D.npz")
except:
print("Cannot read " + savdir + sname + "_2D.npz")
exit()
if beamstop_mask != 'none':
try:
bmask = np.abs(
EdfMethods.loadedf(beamstop_mask)) #reads edf and npy
bmask[bmask > 1] = 1
except:
print("Cannot read beamstop mask %s, skip" % beamstop_mask)
if mask_file != 'none':
try:
mask = np.abs(EdfMethods.loadedf(mask_file)) #reads edf and npy
mask[mask > 1] = 1
try:
mask[bmask > 0] = 1
except:
pass
data[mask > 0] = 0
except:
print("Cannot read mask %s, exit" % mask_file)
exit()
ind = np.where((data > 0) & (mask < 1))
data = np.ma.array(data, mask=mask)
qmask = mask * 0
t0 = time.time()
rad, r_q, new_saxs = tools.radi(data, mask, cx, cy) #radial averaging
print("Calculation time %3.4f sec" % (time.time() - t0))
np.save(savdir + sname + "_gaus.npy", np.array(new_saxs, np.float32))
rad[:, 0] = 4 * np.pi / lambdaw * np.sin(
np.arctan(pix_size * rad[:, 0] / distance) /
2.0) #q vector calculation
r_q = 4 * np.pi / lambdaw * np.sin(
np.arctan(pix_size * r_q / distance) / 2.0) #q vector calculation
width_p = np.tan(
np.arcsin(width_q * lambdaw / 4 / np.pi) * 2) * distance / pix_size
qmask = np.array((r_q - first_q + width_q / 2) / width_q + 1, np.uint16)
print("Number of Qs %d" % number_q)
print("Width of ROI is %1.1f pixels" % width_p)
np.savetxt(savdir + sname + "_1D.dat", rad)
#qmask = np.array((r_q-first_q+width_q/2)/width_q+1,np.uint16)
#qmask[mask>0] = 0
#np.save(savdir+sname+"_qmask.npy",np.array(qmask,np.uint16))
#qmask[qmask>number_q] = 0
#qp = np.linspace(first_q,first_q+(number_q-1)*width_q,number_q)
qp = np.linspace(first_q, first_q + (number_q - 1) * step_q, number_q)
print("Q values = ", qp)
qmask = mask * 0
i_qp = qp * 0
i_bands = []
n = 0
for i in qp:
i_qp[n] = rad[(np.abs(rad[:, 0] - i) == (np.abs(rad[:, 0] - i).min())),
1]
ind1 = np.where((rad[:, 0] >= i - width_q / 2)
& (rad[:, 0] <= i + width_q / 2))[0]
i_bands.append(ind1)
n += 1
indq = np.where(np.abs(r_q - i) <= width_q / 2)
qmask[indq] = n
print("Number of pixels per q %d %d" % (n, qmask[indq].size))
qmask[mask > 0] = 0
np.save(savdir + sname + "_qmask.npy", np.array(qmask, np.uint16))
qmask[qmask > number_q] = 0
qmask = np.ma.array(qmask)
qmask[qmask == 0] = np.ma.masked
new_saxs = np.ma.array(new_saxs, mask=mask)
err = np.abs(data - new_saxs)[ind].mean() / np.mean(data[ind])
print("SAXS and radi modulus error %f" % err)
print("Total calculation time %3.4f sec" % (time.time() - t0))
plt.figure()
plt.imshow(np.log10(new_saxs), cmap='jet')
plt.title("Radi")
plt.figure()
plt.imshow(qmask.data, cmap='gray_r')
plt.plot(cx, cy, 'r+')
plt.grid()
plt.xlabel("Pixel x")
plt.ylabel("Pixel y")
plt.title("Q mask")
plt.figure()
plt.imshow(np.log10(data), cmap='jet')
plt.imshow(qmask, cmap='gray_r', alpha=0.5)
plt.plot(cx, cy, 'r+')
plt.grid()
plt.xlabel("Pixel x")
plt.ylabel("Pixel y")
plt.title("Q mask")
plt.figure()
plt.loglog(rad[:, 0], rad[:, 1], '-b')
plt.loglog(qp, i_qp, 'or', mfc='none')
for n in range(number_q):
if (n + 1) // 2 == (n + 1) / 2:
plt.fill_between(rad[i_bands[n], 0],
rad[i_bands[n], 1],
facecolor='lime',
alpha=0.5)
else:
plt.fill_between(rad[i_bands[n], 0],
rad[i_bands[n], 1],
facecolor='tomato',
alpha=0.5)
plt.xlabel(r"Q ($\AA^{-1}$)")
plt.ylabel("I (counts)")
plt.title(sname)
plt.show()
exit()
def main():
################# READING CONFIG FILE ##########################################################################
try:
config.read(sys.argv[1])
except:
print("Cannot read " + sys.argv[1] + " input file")
exit()
#### Sample description ######################################################################################
sname = config["sample_description"]["name"]
#### Data location ######################################################################################
datdir = config["data_location"]["data_dir"]
sample_dir = config["data_location"]["sample_dir"]
prefd = config["data_location"]["data_prefix"]
sufd = config["data_location"]["data_sufix"]
nf1 = int(config["data_location"]["first_file"])
nf2 = int(config["data_location"]["last_file"])
darkdir = config["data_location"]["dark_dir"]
df1 = int(config["data_location"]["first_dark"])
df2 = int(config["data_location"]["last_dark"])
savdir = config["data_location"]["result_dir"]
#### Experimental setup ######################################################################################
geometry = config["exp_setup"]["geometry"]
cx = float(config["exp_setup"]["dbx"])
cy = float(config["exp_setup"]["dby"])
dt = float(config["exp_setup"]["lagtime"])
lambdaw = float(config["exp_setup"]["wavelength"])
distance = float(config["exp_setup"]["detector_distance"])
first_q = float(config["exp_setup"]["firstq"])
width_q = float(config["exp_setup"]["widthq"])
number_q = int(config["exp_setup"]["numberq"])
beamstop_mask = config["exp_setup"]["beamstop_mask"]
#### Correlator info ######################################################################################
correlator = config["correlator"]["method"]
lth = int(config["correlator"]["low_threshold"])
bADU = int(config["correlator"]["bottom_ADU"])
tADU = int(config["correlator"]["top_ADU"])
mNp = float(config["correlator"]["max_number"])
aduph = int(config["correlator"]["photon_ADU"])
ttcf_par = int(config["correlator"]["ttcf"])
#### Detector description ######################################################################################
detector = config["detector"]["det_name"]
pix_size = float(config["detector"]["pixels"])
mask_file = config["detector"]["mask"]
flatfield_file = config["detector"]["flatfield"]
###################################################################################################################
try:
data = readdata.readnpz(savdir + sname + "_2D.npz")
except:
print("Cannot read " + savdir + sname + "_2D.npz")
exit()
if beamstop_mask != 'none':
try:
bmask = np.abs(
EdfMethods.loadedf(beamstop_mask)) #reads edf and npy
bmask[bmask > 1] = 1
except:
print("Cannot read beamstop mask %s, skip" % beamstop_mask)
if mask_file != 'none':
try:
mask = np.abs(EdfMethods.loadedf(mask_file)) #reads edf and npy
mask[mask > 1] = 1
try:
mask[bmask > 0] = 1
except:
pass
data[mask > 0] = 0
except:
print("Cannot read mask %s, exit" % mask_file)
exit()
################################################
print("Scipy center of mass of raw data x=%3.2f, y=%3.2f" %
center_of_mass(data)[::-1])
t0 = time.time()
cx, cy = tools.beam_center(
data, mask, cx, cy, lc=30,
lw=10) #change lc and lw according to the needs lc>lw
print("Calculation time t=%2.3f sec." % (time.time() - t0))
plt.show()
exit()
def main():
################# READING CONFIG FILE #########################################################
try:
config.read(sys.argv[1])
except:
print("Cannot read " + sys.argv[1] + " input file")
exit()
#### Sample description #######################################################################
sname = config["sample_description"]["name"]
scan = config["sample_description"]["scan"]
scan = str(int(scan.split("scan")[1]))
#### Data location ############################################################################
datdir = config["data_location"]["data_dir"]
sample_dir = config["data_location"]["sample_dir"]
prefd = config["data_location"]["data_prefix"]
sufd = config["data_location"]["data_sufix"]
nf1 = int(config["data_location"]["first_file"])
nf2 = int(config["data_location"]["last_file"])
darkdir = config["data_location"]["dark_dir"]
df1 = int(config["data_location"]["first_dark"])
df2 = int(config["data_location"]["last_dark"])
savdir = config["data_location"]["result_dir"]
toplot = config["data_location"]["toplot"]
#### Experimental setup #######################################################################
geometry = config["exp_setup"]["geometry"]
cx = float(config["exp_setup"]["dbx"])
cy = float(config["exp_setup"]["dby"])
dt = float(config["exp_setup"]["lagtime"])
lambdaw = float(config["exp_setup"]["wavelength"])
distance = float(config["exp_setup"]["detector_distance"])
first_q = float(config["exp_setup"]["firstq"])
width_q = float(config["exp_setup"]["widthq"])
step_q = float(config["exp_setup"]["stepq"])
number_q = int(config["exp_setup"]["numberq"])
qmask_file = config["exp_setup"]["q_mask"]
beamstop_mask = config["exp_setup"]["beamstop_mask"]
#### Correlator info #########################################################################
correlator = config["correlator"]["method"]
engine = config["correlator"]["engine"]
lth = int(config["correlator"]["low_threshold"])
bADU = int(config["correlator"]["bottom_ADU"])
tADU = int(config["correlator"]["top_ADU"])
mNp = float(config["correlator"]["max_number"])
aduph = int(config["correlator"]["photon_ADU"])
ttcf_par = int(config["correlator"]["ttcf"])
#### Detector description ####################################################################
detector = config["detector"]["det_name"]
pix_size = float(config["detector"]["pixels"])
mask_file = config["detector"]["mask"]
flatfield_file = config["detector"]["flatfield"]
#### check if the saving direftory exist and create one #####
tools.test_dir(savdir)
#### Copy config file to the result directory #####
try:
shutil.copy(sys.argv[1], savdir + "input_xpcs_" + sname + ".txt")
except:
pass
###############################################################################################
from dynamix.plot.draw_result import plot_cf, show_trc
import pylab as plt
print("Start analysis of the sample %s" % sname)
print("Running %s correlator using %s processor." % (correlator, engine))
if correlator == "event":
###### event ######################
from dynamix.correlator.event_y import ecorrts, nbecorrts_q, ecorrts_q, gpu_ecorr, gpu_ecorr_q
### read detector mask ###
mask = tools.read_det_mask(mask_file, detector)
### read beamstop mask ####
bmask = tools.read_beamstop_mask(beamstop_mask)
mask[bmask > 0] = 1
### make q ###
#tools.make_q(config)
### read qmask ####
qqmask = tools.read_qmask(qmask_file, mask, number_q)
qqmask[mask > 0] = 0
#### read and dropletize data ####
if detector == "Andor":
#pixels - list of pixels that see a photon
#s - sum of all photons in a frame
#for_norm - number of pixels used
#img - summed 2D scattering pattern
print("Detector Andor")
#get the delta value from the tenth image to adjust cx position and first_q
delta = readdata.get_delta(sample_dir, prefd, sufd, nf1, nf2)
cx = int(cx - distance * np.tan(np.deg2rad(delta)) / pix_size)
first_q = 4 * np.pi / lambdaw * np.sin(np.deg2rad(delta) / 2)
print("Automatic ajusting cx=%d and first_q=%3.3f" % (cx, first_q))
if engine == "CPUold":
pixels, s, for_norm, img = readdata.get_ccd_event_data(
sample_dir, prefd, sufd, nf1, nf2, darkdir, df1, df2,
sname, lth, bADU, tADU, mNp, aduph, savdir, mask_file)
else:
events, times, counter, img, nframes, mask, trace = readdata.get_ccd_event_datan(
sample_dir, prefd, sufd, nf1, nf2, darkdir, df1, df2,
sname, lth, bADU, tADU, mNp, aduph, savdir, mask_file, 20,
0.33)
if engine == "GPU":
times = np.array(times, np.int32)
offsets = np.cumsum(counter, dtype=np.uint32)
offsets = np.insert(offsets, 0, 0)
max_nnz = counter.max() + 1
from dynamix.correlator.event import FramesCompressor
shape = np.shape(img)
dtype = np.int8
F = FramesCompressor(shape, nframes, max_nnz)
elif prefd.find("master") >= 0:
#pixels,s,img,nframes,mask = h5reader.p10_eiger_event_data(sample_dir+prefd+sufd,nf1,nf2,mask)
pixels, s, img, nframes, mask = h5reader.p10_eiger_event_dataf(
sample_dir + prefd + sufd, nf1, nf2, mask, mNp)
trace = np.array(s, np.uint16)
np.savetxt(savdir + sname + "_trace.dat", trace)
sufd = ".5"
print("Number of frames %d" % nframes)
elif sufd == ".h5":
delta = readdata.get_delta(sample_dir, prefd, sufd, nf1, nf2, scan)
ccx = int(1024 + distance * np.tan(np.deg2rad(delta)) / pix_size)
sq = 4 * np.pi / lambdaw * np.sin(np.deg2rad(delta / 2))
print("Delta=%2.2f, suggested cx=%d, central q=%1.3f 1/A" %
(delta, ccx, sq))
if engine == "CPUold":
pixels, s, img, nframes, mask = h5reader.id10_eiger4m_event_dataf(
sample_dir + prefd + sufd, nf1, nf2, mask, mNp, scan)
trace = np.array(s, np.uint16)
np.savetxt(savdir + sname + "_trace.dat", trace)
#if engine == "CCPU":
# mNp = trace.max()+10
# print("Diagnostics maximum photons per frame %d" % (mNp-10))
# pixels, s = tools.events(data, mNp)
if engine == "CPU":
events, times, counter, img, nframes, mask, trace = h5reader.id10_eiger4m_event_GPU_datan(
sample_dir + prefd + sufd, nf1, nf2, mask, scan, 20, 0.33)
np.savetxt(savdir + sname + "_trace.dat", trace)
t0 = time.time()
qqmask[mask > 0] = 0
print("Diagnostics max value %d" % events.max())
print("Counter max value %d" % counter.max())
events = np.array(events, np.int8)
print("Events", events.shape, events.dtype,
events.nbytes // 1024**2)
print("Times", times.shape, times.dtype,
times.nbytes // 1024**2)
print("Counter", counter.shape, counter.dtype,
counter.nbytes // 1024**2)
print("Preprocessing time %f" % (time.time() - t0))
if engine == "GPU":
events, times, counter, img, nframes, mask, trace = h5reader.id10_eiger4m_event_GPU_datan(
sample_dir + prefd + sufd, nf1, nf2, mask, scan, 20, 0.33)
np.savetxt(savdir + sname + "_trace.dat", trace)
t0 = time.time()
qqmask[mask > 0] = 0
print("Diagnostics max value %d" % events.max())
print("Counter max value %d" % counter.max())
events = np.array(events, np.int8)
print("Events", events.shape, events.dtype,
events.nbytes // 1024**2)
print("Times", times.shape, times.dtype,
times.nbytes // 1024**2)
print("Counter", counter.shape, counter.dtype,
counter.nbytes // 1024**2)
print("Preprocessing time %f" % (time.time() - t0))
times = np.array(times, np.int32)
offsets = np.cumsum(counter, dtype=np.uint32)
offsets = np.insert(offsets, 0, 0)
max_nnz = counter.max() + 1
from dynamix.correlator.event import FramesCompressor
shape = np.shape(img)
dtype = np.int8
F = FramesCompressor(shape, nframes, max_nnz)
try:
del data
except:
pass
else:
if engine == "CPUold":
pixels, s, for_norm, img = readdata.get_eiger_event_data(
sample_dir, prefd, sufd, nf1, nf2, sname, mNp, savdir,
mask_file)
else:
events, times, counter, img, nframes, mask, trace = readdata.get_eiger_event_datan(
sample_dir, prefd, sufd, nf1, nf2, sname, mNp, savdir,
mask_file, 20, 0.33)
if engine == "GPU":
times = np.array(times, np.int32)
offsets = np.cumsum(counter, dtype=np.uint32)
offsets = np.insert(offsets, 0, 0)
max_nnz = counter.max() + 1
from dynamix.correlator.event import FramesCompressor
shape = np.shape(img)
dtype = np.int8
F = FramesCompressor(shape, nframes, max_nnz)
#### save summed image #####
np.savez_compressed(savdir + sname + "_2D_raw.npz",
data=np.array(img, np.float32))
### apply mask ###
img[mask > 0] = 0
### read and apply flat field ###
if flatfield_file != 'none':
try:
flatfield = EdfMethods.loadedf(flatfield_file)
flatfield[flatfield < 0] = 1
except:
print("Cannot read flat field %s skip" % flatfield_file)
flatfield = np.ones(mask.shape, np.float32)
else:
flatfield = np.ones(mask.shape, np.float32)
np.savez_compressed(savdir + sname + "_2D.npz",
data=np.array(img / flatfield, np.float32))
### qmask #############
#qqmask = mask*0+1
#qqmask[mask>0] = 0
### make q ###
tools.make_q(config)
### read qmask ####
qqmask = tools.read_qmask(qmask_file, mask, number_q)
cdata = tools.make_cdata(savdir + sname + "_gaus.npy", config)
### correlate ##########
if engine == "CPUold":
CorrelationResult = ecorrts_q(pixels, s, qqmask, True, ttcf_par)
if engine == "CPU":
CorrelationResult = nbecorrts_q(events, times, counter, qqmask,
nframes, True, ttcf_par)
if engine == "GPU":
#CorrelationResult = gpu_ecorr(events, times, offsets, shape, nframes, dtype, qqmask, F)
CorrelationResult = gpu_ecorr_q(events, times, offsets, shape,
nframes, dtype, qqmask, max_nnz)
### format result #######
res, save_cf, trc = tools.format_result(CorrelationResult, qqmask,
flatfield, cdata, dt, ttcf_par)
### save cf ##############
qp = np.linspace(first_q, first_q + (number_q - 1) * step_q, number_q)
tools.save_cf(savdir + sname + "_event_cf.dat", save_cf, qp)
#### plot results #######
if toplot == "yes":
try:
plot_cf(res, sname)
except:
pass
print("Saving trc")
try:
if trc.size > 1:
readdata.savenpz(savdir + sname + "_event_trc.npz",
np.array(trc, np.float32))
except:
pass
if toplot == "yes":
try:
print("Ploting trc, please wait")
show_trc(trc, sname, savdir)
except:
pass
elif correlator == "intensity":
##### intensity #################################
from dynamix.correlator.dense import MatMulCorrelator
from dynamix.correlator.cuda import CublasMatMulCorrelator
##### read data ##########################################
if sufd.find("edf") > -1: #== ".edf":
data = readdata.get_data(sample_dir, prefd, sufd, nf1,
nf2) #[:3000,:,:]
elif sufd == ".h5":
data = h5reader.myreader(sample_dir + prefd + sufd)[nf1:nf2, :, :]
data = np.array(data, np.uint8)
else:
exit()
img = np.array(np.mean(data, 0), np.float32)
if os.path.isfile(savdir + sname + "_2D_raw.npz"):
pass
else:
np.savez_compressed(savdir + sname + "_2D_raw.npz", data=img)
### read detector mask ###
mask = EdfMethods.loadedf(mask_file)
### read beamstop mask ####
bmask = tools.read_beamstop_mask(beamstop_mask)
mask[bmask > 0] = 1
img[mask > 0] = 0
if flatfield_file != 'none':
try:
flatfield = EdfMethods.loadedf(flatfield_file)
flatfield[flatfield < 0] = 1
except:
print("Cannot read flat field %s skip" % flatfield_file)
flatfield = np.ones(mask.shape, np.float32)
else:
flatfield = np.ones(mask.shape, np.float32)
if os.path.isfile(savdir + sname + "_2D.npz"):
pass
else:
np.savez_compressed(savdir + sname + "_2D.npz",
data=np.array(img / flatfield, np.float32))
print("Original data size is %2.2f Gigabytes" %
(data.size * data.itemsize / 1024**3))
print("Original data type", data.dtype, data.max())
### make smooth data ####
cdata = tools.make_cdata(savdir + sname + "_gaus.npy", config)
### read qmask ####
qqmask = tools.read_qmask(qmask_file, mask, number_q)
#### reduce the matrix size to speed up calculations ####
data, qqmask, cdata, flatfield = tools.reduce_matrix(
data, qqmask, cdata, flatfield)
shape = np.shape(data[0, :, :])
nframes = np.shape(data)[0]
print("Number of frames %d" % nframes)
print("Number of qs %d" % number_q)
### correlator #######
t0 = time.time()
#### MatMul Correlator #####################################
if engine == "CPU":
correlator = MatMulCorrelator(shape, nframes, qmask=qqmask)
if engine == "GPU":
correlator = CublasMatMulCorrelator(shape, nframes, qmask=qqmask)
t1 = time.time()
result = correlator.correlate(data)
print("Correlator time %3.2f seconds" % (time.time() - t1))
trace = np.zeros((nframes, number_q), np.float32)
res = []
n = 0
cf = np.zeros((nframes - 1, 3), np.float32)
cf[:, 0] = np.arange(1, nframes, 1) * dt
for q in range(1, number_q + 1, 1):
trace[:, n] = np.sum(data[:, qqmask == q], 1)
fcorrection = (flatfield[qqmask == q]**
2).mean() / flatfield[qqmask == q].mean()**2
correction = (cdata[qqmask == q]**2
).mean() / cdata[qqmask == q].mean()**2 * fcorrection
if engine == "GPU":
cf[:, 1] = result[n][1:] / correction #GPU
cf[:, 2] = result[n][1:] / correction * 1e-4 #GPU
if engine == "CPU":
cf = result[n] #CPU
cf[:, 0] = cf[:, 0] * dt #CPU
cf[:, 1] /= correction # make correct baseline#CPU
cf[:, 2] /= correction # make correct baseline#CPU
cfm = tools.cftomt(cf)
res.append(cfm)
if q == 1:
save_cf = cfm
else:
save_cf = np.append(save_cf, cfm[:, 1:], axis=1)
n += 1
print("Correlation time %3.2f seconds" % (time.time() - t0))
### save cf ##############
qp = np.linspace(first_q, first_q + (number_q - 1) * step_q, number_q)
tools.save_cf(savdir + sname + "_event_cf.dat", save_cf, qp)
### save trace ####
q_title = '#q values 1/A:'
for q in qp:
q_title = q_title + " %.5f " % q
q_title = q_title + '\n'
f = open(savdir + sname + "_trace.dat", 'w')
f.write(q_title)
np.savetxt(f, trace)
f.close()
#np.savetxt(savdir+sname+"_trace.dat",trace)
#np.savetxt(savdir+sname+"_cf.dat",save_cf)
#readdata.savenpz(savdir+sname+"_trc.npz",np.array(trc/correction,np.float32))
#### plot results #######
if toplot == "yes":
try:
plot_cf(res, sname)
except:
pass
try:
show_trc(trc, sname, savdir)
except:
pass
else:
pass
exit()
def get_ccd_event_datan(datdir,
prefd,
sufd,
nf1,
nf2,
darkdir,
df1,
df2,
sname,
lth,
bADU,
tADU,
mNp,
aduph,
savdir,
mask_file,
thr=20,
frc=0.15):
### read ccd edf images and convert for Pierre's event correlator using numba ####
t0 = time.time()
swrite = stdout.write
sflush = stdout.flush
print("start reading the files")
# creating filenames
filenames = nfiles.filename(datdir + prefd, sufd, nf1, nf2)
lfilenames = len(filenames) # -1
# reading first image to get dimenstions of the matrix
headers = EdfMethods.headeredf(filenames[0])
dim1 = np.intc(headers['Dim_1'])
dim2 = np.intc(headers['Dim_2'])
nx = dim2
ny = dim1
############reading mask##########
try:
mask_data = EdfMethods.loadedf(mask_file)
print("use mask file " + mask_file)
ind = np.where(mask_data > 0)
for_norm = dim1 * dim2 - len(mask_data[ind])
except:
mask_data = np.zeros((dim2, dim1), np.uint8)
print("no mask applied")
for_norm = dim1 * dim2 # 1024**2
pass
print("Numebr of pixels used " + str(for_norm))
########reading dark##########
darkfilenames = nfiles.filename(darkdir + prefd, sufd, df1, df2)
ndarks = 0
for dfile in darkfilenames:
if ndarks == 0:
darkimg = np.asfarray(EdfMethods.loadedf(dfile), dtype=np.float32)
else:
darkimg += np.asfarray(EdfMethods.loadedf(dfile), dtype=np.float32)
ndarks += 1
darkimg = darkimg / ndarks
n_frames = len(filenames)
ll = nx * ny # total number of pixels
lp = int(n_frames * frc) # total number of frames with events 15%
mask = np.array(np.ravel(mask_data), np.uint8)
evs = np.zeros((ll, lp), np.uint8)
tms = np.zeros((ll, lp), np.uint16)
cnt = np.ravel(np.zeros((ll, ), np.uint16))
afr = np.ravel(np.zeros((ll, ), np.uint32))
tr = 0
trace = np.zeros((n_frames, ), np.uint32)
it = 0
print("Number of frames %d" % n_frames)
###########reading and summing files###########
for i in range(lfilenames):
swrite(4 * '\x08')
swrite(str(int(i * 100. / lfilenames)) + '%')
sflush()
matr = np.asfarray(EdfMethods.loadedf(filenames[i]), dtype=np.float32)
try:
matr[ind] = 0
except:
pass
msumpix, mpix, fr = dropimgood(matr, darkimg, lth, bADU, tADU, mNp,
aduph, nx, ny) # dropletize CCD frames
fr = np.ravel(fr)
evs, tms, cnt, afr, mask, tr = neigercompress(evs, tms, cnt, afr, mask,
tr, fr, thr, it, ll, lp)
trace[i] = tr
it += 1
if not os.path.exists(savdir):
answ = input("create a director (y)/n").lower()
if answ == "n":
print("exit")
exit()
else:
os.makedirs(savdir)
print("directory " + savdir + " has been created")
afr = afr / n_frames
afr = np.reshape(afr, (nx, ny))
mask = np.reshape(mask, (nx, ny))
evs, tms, c = nprepare(np.ravel(evs), np.ravel(tms))
evs = np.array(evs[:c], np.int8)
tms = tms[:c]
print("Reading time %3.3f sec" % (time.time() - t0))
return evs, tms, cnt, afr, n_frames, mask, trace
def get_ccd_event_data(datdir, prefd, sufd, nf1, nf2, darkdir, df1, df2, sname,
lth, bADU, tADU, mNp, aduph, savdir, mask_file):
### read ccd edf images and convert for event correlator ####
time0 = time.time()
swrite = stdout.write
sflush = stdout.flush
print("start reading the files")
# creating filenames
filenames = nfiles.filename(datdir + prefd, sufd, nf1, nf2)
lfilenames = len(filenames) # -1
# reading first image to get dimenstions of the matrix
headers = EdfMethods.headeredf(filenames[0])
dim1 = np.intc(headers['Dim_1'])
dim2 = np.intc(headers['Dim_2'])
nx = dim2
ny = dim1
############reading mask##########
try:
mask_data = EdfMethods.loadedf(mask_file)
print("use mask file " + mask_file)
ind = np.where(mask_data > 0)
for_norm = dim1 * dim2 - len(mask_data[ind])
except:
print("no mask applied")
for_norm = dim1 * dim2 # 1024**2
pass
print("Numebr of pixels used " + str(for_norm))
########creating image matrix of zeros############
img = np.zeros((dim2, dim1))
########reading dark##########
darkfilenames = nfiles.filename(darkdir + prefd, sufd, df1, df2)
ndarks = 0
for dfile in darkfilenames:
if ndarks == 0:
darkimg = np.asfarray(EdfMethods.loadedf(dfile), dtype=np.float32)
else:
darkimg += np.asfarray(EdfMethods.loadedf(dfile), dtype=np.float32)
ndarks += 1
darkimg = darkimg / ndarks
pixels = []
s = []
data = []
pread = []
for i in range(2):
data.append(Queue(2))
# pread.append(Process(target=mread_ccd, args=(filenames[i:-1:2],data[i],darkimg,lth,bADU,tADU,mNp,aduph,nx,ny)))
pread.append(
Process(target=mread_ccd,
args=(filenames[:-1:2], data[0], darkimg, lth, bADU, tADU, mNp,
aduph, nx, ny)))
pread.append(
Process(target=mread_ccd,
args=(filenames[1::2], data[1], darkimg, lth, bADU, tADU, mNp,
aduph, nx, ny)))
for i in range(2):
pread[i].start()
ii = 0
###########reading and summing files###########
for i in range(lfilenames):
swrite(4 * '\x08')
swrite(str(int(i * 100. / lfilenames)) + '%')
sflush()
fromproc = data[ii].get()
pixels.append(fromproc[0])
s.append(fromproc[1])
img += fromproc[2]
ii += 1
if ii == 2:
ii = 0
for i in range(2):
pread[i].join()
data[i].close()
dtime = time.time() - time0
print('reading of %d files took %5.2f sec' % (lfilenames, dtime))
if not os.path.exists(savdir):
answ = input("create a director (y)/n")
if answ == "n":
print("exit")
exit()
else:
os.makedirs(savdir)
print("directory " + savdir + " has been created")
return pixels, s, for_norm, img