def run(infile):
h5 = h5py.File(infile, "r")
if h5["/entry/instrument/detector/flatfield_correction_applied"].value:
print "Correction was already applied."
return
ff = h5["/entry/instrument/detector/detectorSpecific/flatfield"][:]
outdir = "flatfield_applied_%s" % os.path.splitext(os.path.basename(infile))[0]
if not os.path.exists(outdir): os.mkdir(outdir)
cbf.save_numpy_data_as_cbf((ff*1000).astype(numpy.uint32).flatten(), size1=ff.shape[1], size2=ff.shape[0], title="flatfield_x1000",
cbfout=os.path.join(outdir, "flatfield.cbf"))
for k in sorted(h5["/entry/data"].keys()):
outfile = os.path.join(outdir, h5["/entry/data"].get(k, getlink=True).filename)
ds = h5["/entry/data"][k]
h5o = h5py.File(outfile, "w")
h5o.create_group("/entry")
h5o["/entry"].attrs["NX_class"] = "NXentry"
h5o.create_group("/entry/data")
h5o["/entry/data"].attrs["NX_class"] = "NXdata"
dataset = h5o.create_dataset("/entry/data/data", ds.shape,
compression=bitshuffle.h5.H5FILTER,
compression_opts=(0, bitshuffle.h5.H5_COMPRESS_LZ4),
chunks=ds.chunks, dtype=ds.dtype)
dataset.attrs["image_nr_low"] = ds.attrs["image_nr_low"]
dataset.attrs["image_nr_high"] = ds.attrs["image_nr_high"]
# Don't want to load all data to memory at the same time
for i in xrange(ds.shape[0]):
print "processing %6d/%d in %s" % (i+1, ds.shape[0], k)
dataset[i,] = ds[i,] * ff + .5
def run(mrcin, prefix=None):
f = mrcfile.open(mrcin)
if prefix is None: prefix = os.path.splitext(os.path.basename(mrcin))[0]
data = f.data
if data.ndim == 2: data = data.reshape(-1, *data.shape)
print "MRC file loaded"
print " shape = %s" % (data.shape,)
print " dtype = %s" % data.dtype
print
for i in xrange(data.shape[0]):
size2, size1 = data[i].shape # XXX really?
cbfout = "%s_%.3d.cbf"%(prefix, i+1)
print "Writing %s" % cbfout
cbf.save_numpy_data_as_cbf(data[i].flatten(), size1, size2,
"%s:%d"%(mrcin, i+1),
cbfout, pilatus_header="""
# Detector: %(detname)s
# Pixel_size %(pixelsize)e m x %(pixelsize)e m
# Wavelength %(wavelength)f A
# Detector_distance %(distance)f m
# Beam_xy (%(beamx)f, %(beamy)f) pixels
# Start_angle %(start_angle)%.4f deg.
# Angle_increment %(angle_inc).4f deg.
""" % dict(detname="????",
pixelsize=75.e-6,
wavelength=1,
distance=300.e-3,
beamx=size1/2., beamy=size2/2.,
start_angle=0, angle_inc=0))
def run(params):
xmin, xmax, nxbin, positions = read_correct_lp(params.correct_lp)
xstep = (xmax-xmin)/nxbin
absorp = read_absorp_cbf(params.absorp_cbf)
assert numpy.all(absorp > 13) # if
det = read_bkgpix_cbf(params.bkgpix_cbf, positions)
datout = open("%s.dat"%params.output.prefix, "w")
datout.write("ix xmin xmax ipos posx posy fac\n")
for ix in xrange(nxbin):
x1, x2 = xmin + ix*xstep, xmin + (ix+1)*xstep
tmp = numpy.array(det)
for i in xrange(len(positions)):
tmp[tmp==i+1] = absorp[i,ix]
datout.write("%2d %.2f %.2f %2d %d %d %5d\n" % (ix, x1, x2, i, positions[i][0], positions[i][1], absorp[i,ix]))
tmp = numpy.array(tmp, dtype=numpy.int32)
cbf.save_numpy_data_as_cbf(tmp.flatten(), tmp.shape[1], tmp.shape[0],
"absorp_%.1f-%.1f"% (x1,x2),
"%s_%.3d.cbf" % (params.output.prefix, ix))
datout.close()
def save_cbf(wavelen, data, cbfout):
header = """\
# Detector: NOT PILATUS but MPCCD
# Wavelength %f A
""" % wavelen
height, width = data.shape
#print "Saving", cbfout, height, width, data
cbf.save_numpy_data_as_cbf(data.reshape(width*height), width, height, "hdf5_converted", str(cbfout),
pilatus_header=header)
def run(cbfin, cbfout, repl=None):
data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(cbfin)
max_I = max(data)
if repl is None:
repl = max_I+100
data[data<0] = repl
cbf.save_numpy_data_as_cbf(data, ndimfast, ndimmid, "negative_replaced", cbfout)
print "Launch:"
print "adxv -overload %d %s" % (max_I, cbfout)
def run(cbfin, cbfout, repl=None):
data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(cbfin)
max_I = max(data)
if repl is None:
repl = max_I + 100
data[data < 0] = repl
cbf.save_numpy_data_as_cbf(data, ndimfast, ndimmid, "negative_replaced",
cbfout)
print "Launch:"
print "adxv -overload %d %s" % (max_I, cbfout)
def run(files, cbfout):
merged = None
for i, f in enumerate(files):
repl = -10 * (i+1)
print "%s %d" % (f, repl)
data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(f)
if i == 0:
merged = data.copy()
continue
merged[data==-10] = 65540 # repl # for adxv visualization. only assuming two files.
cbf.save_numpy_data_as_cbf(merged, ndimfast, ndimmid, "merged_predictions", cbfout)
def extract_to_minicbf(h5master, frameno_or_path, cbfout, binning=1):
from yamtbx.dataproc import cbf
from yamtbx.dataproc.XIO.plugins import eiger_hdf5_interpreter
if type(frameno_or_path) in (tuple, list):
data = extract_data_range_sum(h5master, frameno_or_path)
nframes = len(frameno_or_path)
elif type(frameno_or_path) is int:
data = extract_data(h5master, frameno_or_path)
nframes = 1
else:
data = extract_data_path(h5master, frameno_or_path)
nframes = 1
if data is None:
raise RuntimeError("Cannot extract frame %s from %s" %
(frameno_or_path, h5master))
h = eiger_hdf5_interpreter.Interpreter().getRawHeadDict(h5master)
h5 = h5py.File(h5master, "r")
if binning > 1:
beamxy = h["BeamX"], h["BeamY"]
data, (h["BeamX"],
h["BeamY"]) = software_binning(data, binning, beamxy)
h["Detector"] = "Unknown"
if "/entry/instrument/detector/description" in h: # EIGER2 does not have this field?
h["Detector"] = h5["/entry/instrument/detector/description"].value
h["ExposurePeriod"] = h5["/entry/instrument/detector/frame_time"].value
h["PhiWidth"] *= nframes
cbf.save_numpy_data_as_cbf(data.flatten(),
size1=data.shape[1],
size2=data.shape[0],
title="",
cbfout=cbfout,
pilatus_header="""\
# Detector: %(Detector)s, S/N %(SerialNumber)s
# Pixel_size %(PixelX)e m x %(PixelY)e m
# %(SensorMaterial)s sensor, thickness %(SensorThickness).3e m
# Exposure_time %(ExposureTime).6f s
# Exposure_period %(ExposurePeriod).6f s
# Count_cutoff %(Overload)d counts
# Wavelength %(Wavelength).6f A
# Detector_distance %(Distance).3e m
# Beam_xy (%(BeamX).1f, %(BeamY).1f) pixels
# Start_angle %(PhiStart).6f deg.
# Angle_increment %(PhiWidth).6f deg.
""" % h)
def save_cbf(wavelen, data, cbfout):
header = """\
# Detector: NOT PILATUS but MPCCD
# Wavelength %f A
""" % wavelen
height, width = data.shape
#print "Saving", cbfout, height, width, data
cbf.save_numpy_data_as_cbf(data.reshape(width * height),
width,
height,
"hdf5_converted",
str(cbfout),
pilatus_header=header)
def run(files, cbfout):
merged = None
for i, f in enumerate(files):
repl = -10 * (i + 1)
print "%s %d" % (f, repl)
data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(f)
if i == 0:
merged = data.copy()
continue
merged[
data ==
-10] = 65540 # repl # for adxv visualization. only assuming two files.
cbf.save_numpy_data_as_cbf(merged, ndimfast, ndimmid, "merged_predictions",
cbfout)
def write_minicbf(data, metadata, prefix, frame_range, alpha_step, time_step, params):
detector = str(metadata[0]["BinaryResult"]["Detector"])
voltage = float(metadata[0]["Optics"]["AccelerationVoltage"])
wavelen = electron_voltage_to_wavelength(voltage)
#width = int(metadata[0]["BinaryResult"]["ImageSize"]["width"]) #/ int(metadata[0]["Detectors"]["ImageSize"]["width"])
#height = int(metadata[0]["BinaryResult"]["ImageSize"]["height"]) # / binning!!
width, height = data.shape[:2] # the other way around?
binning = int(metadata[0]["BinaryResult"]["ImageSize"]["width"])//width
exp_time = float(metadata[0]["Detectors"]["Detector-0"]["ExposureTime"])
px = params.pixel_size*1.e-6 * binning
clen = params.clen
if params.beamxy is None:
beamx, beamy = (width/2., height/2.)
else:
beamx, beamy = params.beamxy
print "There are %d frames" % data.shape[2]
for i in xrange(data.shape[2]):
if i < frame_range[0] or i > frame_range[1]:
print "Ignoring frame %d" % (i+1)
continue
print "Converting", i+1
frame_date = datetime.datetime.fromtimestamp(int(metadata[i]["CustomProperties"]["Detectors[%s].TimeStamp"%params.detector]["value"])/1.e6)
datestr = datetime.datetime.strftime(frame_date, "%Y-%m%-dT%H:%M:%S.%f")
start_angle = str(metadata[i]["Stage"]["AlphaTilt"])
cbf.save_numpy_data_as_cbf(data[:,:,i].flatten().astype(numpy.int32)+params.offset, data.shape[0], data.shape[1],
"", "%s_%.6d.cbf"%(prefix, i+1),
pilatus_header="""
# Detector: %(detector)s
# %(datestr)s
# Pixel_size %(px).1e m x %(px).1e m
# Exposure_time %(exp_time)s s
# Exposure_period %(time_step)f s
# Wavelength %(wavelen)f A
# Detector_distance %(clen)f m
# Beam_xy (%(beamx).2f, %(beamy).2f) pixels
# Start_angle %(start_angle)s deg.
# Angle_increment %(alpha_step)f deg.
""" % locals(),
header_convention="PILATUS_1.2")
def run(infile):
h5 = h5py.File(infile, "r")
if h5["/entry/instrument/detector/flatfield_correction_applied"].value:
print "Correction was already applied."
return
ff = h5["/entry/instrument/detector/detectorSpecific/flatfield"][:]
outdir = "flatfield_applied_%s" % os.path.splitext(
os.path.basename(infile))[0]
if not os.path.exists(outdir): os.mkdir(outdir)
cbf.save_numpy_data_as_cbf((ff * 1000).astype(numpy.uint32).flatten(),
size1=ff.shape[1],
size2=ff.shape[0],
title="flatfield_x1000",
cbfout=os.path.join(outdir, "flatfield.cbf"))
for k in sorted(h5["/entry/data"].keys()):
outfile = os.path.join(outdir,
h5["/entry/data"].get(k, getlink=True).filename)
ds = h5["/entry/data"][k]
h5o = h5py.File(outfile, "w")
h5o.create_group("/entry")
h5o["/entry"].attrs["NX_class"] = "NXentry"
h5o.create_group("/entry/data")
h5o["/entry/data"].attrs["NX_class"] = "NXdata"
dataset = h5o.create_dataset(
"/entry/data/data",
ds.shape,
compression=bitshuffle.h5.H5FILTER,
compression_opts=(0, bitshuffle.h5.H5_COMPRESS_LZ4),
chunks=ds.chunks,
dtype=ds.dtype)
dataset.attrs["image_nr_low"] = ds.attrs["image_nr_low"]
dataset.attrs["image_nr_high"] = ds.attrs["image_nr_high"]
# Don't want to load all data to memory at the same time
for i in xrange(ds.shape[0]):
print "processing %6d/%d in %s" % (i + 1, ds.shape[0], k)
dataset[i, ] = ds[i, ] * ff + .5
def extract_to_minicbf(h5master, frameno, cbfout, binning=1):
from yamtbx.dataproc import cbf
from yamtbx.dataproc.XIO.plugins import eiger_hdf5_interpreter
if type(frameno) in (tuple, list):
data = extract_data_range_sum(h5master, frameno)
nframes = len(frameno)
else:
data = extract_data(h5master, frameno)
nframes = 1
if data is None:
raise RuntimeError("Cannot extract frame %s from %s"%(frameno, h5master))
h = eiger_hdf5_interpreter.Interpreter().getRawHeadDict(h5master)
h5 = h5py.File(h5master, "r")
if binning>1:
beamxy = h["BeamX"], h["BeamY"]
data, (h["BeamX"], h["BeamY"]) = software_binning(data, binning, beamxy)
h["Detector"] = h5["/entry/instrument/detector/description"].value
h["ExposurePeriod"] = h5["/entry/instrument/detector/frame_time"].value
h["PhiWidth"] *= nframes
cbf.save_numpy_data_as_cbf(data.flatten(), size1=data.shape[1], size2=data.shape[0], title="",
cbfout=cbfout,
pilatus_header="""\
# Detector: %(Detector)s, S/N %(SerialNumber)s
# Pixel_size %(PixelX)e m x %(PixelY)e m
# %(SensorMaterial)s sensor, thickness %(SensorThickness).3e m
# Exposure_time %(ExposureTime).6f s
# Exposure_period %(ExposurePeriod).6f s
# Count_cutoff %(Overload)d counts
# Wavelength %(Wavelength).6f A
# Detector_distance %(Distance).3e m
# Beam_xy (%(BeamX).1f, %(BeamY).1f) pixels
# Start_angle %(PhiStart).6f deg.
# Angle_increment %(PhiWidth).6f deg.
""" % h)
def run(h5in, cbf_prefix):
f = h5py.File(h5in, "r")
if "instrument" not in f["entry"]:
print "Error: This is not master h5 file."
return
dname = os.path.dirname(cbf_prefix)
if dname != "" and not os.path.exists(dname):
os.makedirs(dname)
print "dir created:", dname
# Analyze pixel_mask
pixel_mask = numpy.array(f["entry"]["instrument"]["detector"]["detectorSpecific"]["pixel_mask"])
print "No. of unuseful pixels:"
for val in set(pixel_mask.flatten()):
if val==0: continue
print "", get_mask_info(val), (pixel_mask==val).sum()
print
# Extract and write data
data = filter(lambda x: x.startswith("data_"), f["entry"])
count = 0
for key in sorted(data):
print "Extracting", key
im = f["entry"][key]
print " shape=", im.shape
print " dtype=", im.dtype
nframes, height, width = im.shape
for i in xrange(nframes):
count += 1
cbfout = "%s_%.6d.cbf" % (cbf_prefix, count)
data = im[i,].astype(numpy.int32)
data[pixel_mask>0] = -1
cbf.save_numpy_data_as_cbf(data.reshape(width*height), width, height, "hdf5_converted", cbfout,
pilatus_header=make_dummy_pilatus_header(f))
print " ", cbfout
print
def on_monitor_timer(self, ev):
print "monitoring"
response = urllib.urlopen("http://%s/monitor/api/%s/images/monitor" % (self.txtEigerHost.GetValue(), self.txtEigerAPIver.GetValue()))
data = response.read()
data_size = len(data)
curlines = self.txtInfo.GetValue().splitlines()
if len(curlines) > 20:
self.txtInfo.SetValue("\n".join(curlines))
if data_size < 10000:
self.txtInfo.SetValue("%s: %s\n%s" % (time.ctime(), data, self.txtInfo.GetValue()))
return
#import cPickle as pickle
#pickle.dump(data, open("monitor.pkl","wb"), -1)
#data = pickle.load(open("monitor.pkl"))
def request(query, base="detector"):
r = urllib.urlopen("http://%s/%s/api/%s/%s"%(self.txtEigerHost.GetValue(), base, self.txtEigerAPIver.GetValue(), query)).read()
return json.loads(r)
nx = request("config/x_pixels_in_detector")["value"]
ny = request("config/y_pixels_in_detector")["value"]
if len(data) < 1000:
data = self.last_monitor_image
if data is None: return
else:
byte = data_size // (nx*ny)
assert byte == 4 or byte == 2
data = numpy.fromstring(data[8:-102], dtype=numpy.int32 if byte==4 else numpy.int16).reshape(ny,nx)
if self.last_monitor_image is not None and (data==self.last_monitor_image).all():
return
wavelen = request("config/wavelength")["value"]
beamx = request("config/beam_center_x")["value"]
beamy = request("config/beam_center_y")["value"]
distance = request("config/detector_distance")
image_number = request("status/monitor_image_number", base="monitor")["value"]
self.txtInfo.SetValue("""\
%s: Downloaded (frame: %.6d)
Wavelength: %.5f A
Detector_distance: %.2f %s
Beam_xy: %.2f, %.2f
%s"""% (time.ctime(), image_number[1]+1, wavelen, distance["value"], str(distance["unit"]), beamx, beamy, self.txtInfo.GetValue()))
from yamtbx.dataproc import cbf
tmpdir = "/dev/shm" if os.path.isdir("/dev/shm") else tempfile.gettempdir()
imgfile = os.path.join("/dev/shm", "adxvtmp-%s-%s.cbf"%(getpass.getuser(), os.getpid()))
xp = request("config/x_pixel_size")
yp = request("config/y_pixel_size")
cbf.save_numpy_data_as_cbf(data.flatten(), size1=data.shape[1], size2=data.shape[0], title="",
cbfout=imgfile,
pilatus_header="""\
# Detector: Eiger
# Pixel_size %(px).5e %(pxu)s x %(py).5e %(pyu)s
# Wavelength %(Wavelength).6f A
# Detector_distance %(Distance).4e %(DistanceU)s
# Beam_xy (%(BeamX).1f, %(BeamY).1f) pixels
""" % dict(px=xp["value"], pxu=str(xp["unit"]), py=yp["value"], pyu=str(yp["unit"]), Wavelength=wavelen, Distance=distance["value"], DistanceU=str(distance["unit"]), BeamX=beamx, BeamY=beamy))
self.adxv.open_image(imgfile, raise_window=self.chkMonRaiseW.GetValue())
self.last_monitor_image = data
def run(xds_inp):
xp = XPARM()
xp.set_info_from_xdsinp_or_inpstr(xdsinp=xds_inp)
d_map = numpy.zeros((xp.ny, xp.nx)) # resolution mapping
ed = numpy.zeros((3, 3))
ed[:, 0] = xp.X_axis
ed[:, 1] = xp.Y_axis
ed[:, 2] = numpy.cross(xp.X_axis, xp.Y_axis)
ed /= numpy.linalg.norm(ed, axis=0)
orgx, orgy = xp.origin
fs = xp.distance
qx, qy = xp.qx, xp.qy
org = -orgx * qx * ed[:, 0] - orgy * qy * ed[:, 1] + fs * ed[:, 2]
wavelen = xp.wavelength
s0 = xp.incident_beam / numpy.linalg.norm(xp.incident_beam) / wavelen
print "qx,qy=", qx, qy
print "s0=", s0
print "lambda=", wavelen
print "ORG=", org
print "ED="
print ed
ofs_pdb = open("detector_pos.pdb", "w")
ofs_pml = open("for_pymol.pml", "w")
for iseg, seg in enumerate(xp.segments):
print("Segment %d" % (iseg + 1))
eds = numpy.zeros((3, 3))
eds[:, 0] = seg.eds_x
eds[:, 1] = seg.eds_y
eds[:, 2] = numpy.cross(seg.eds_x, seg.eds_y)
eds /= numpy.linalg.norm(eds, axis=0)
edsl = numpy.dot(ed, eds)
ix, iy = numpy.meshgrid(range(seg.x1 - 1, seg.x2),
range(seg.y1 - 1, seg.y2))
tmp = numpy.zeros((ix.shape[0], ix.shape[1], 3))
for i in range(3):
tmp[:, :, i] = qx * (ix - seg.orgxs) * edsl[i, 0] + qy * (
iy - seg.orgys) * edsl[i, 1] + seg.fs * edsl[i, 2] + org[i]
write_atom(ofs_pdb, tmp, iseg)
ofs_pml.write("bond resi %d and name C1, resi %d and name C2\n" %
(iseg, iseg))
ofs_pml.write("bond resi %d and name C2, resi %d and name C4\n" %
(iseg, iseg))
ofs_pml.write("bond resi %d and name C3, resi %d and name C4\n" %
(iseg, iseg))
ofs_pml.write("bond resi %d and name C3, resi %d and name C1\n" %
(iseg, iseg))
tmpdenom = numpy.linalg.norm(tmp, axis=2) * wavelen
for i in range(3):
tmp[:, :, i] /= tmpdenom
s1 = tmp
s = s1 - s0
d_map[iy, ix] = 1. / numpy.linalg.norm(s, axis=2)
"""
for ix in range(seg.x1-1, seg.x2):
for iy in range(seg.y1-1, seg.y2):
tmp = (qx*(ix-seg.orgxs)*edsl[0,0]+qy*(iy-seg.orgys)*edsl[0,1]+seg.fs*edsl[0,2]+orgx,
qx*(ix-seg.orgxs)*edsl[1,0]+qy*(iy-seg.orgys)*edsl[1,1]+seg.fs*edsl[1,2]+orgy,
qx*(ix-seg.orgxs)*edsl[2,0]+qy*(iy-seg.orgys)*edsl[2,1]+seg.fs*edsl[2,2]+fs)
s1 = tmp/numpy.linalg.norm(tmp)/wavelen
s = s1-s0
d_map[iy,ix] = numpy.linalg.norm(s)
"""
ofs_pml.write("pseudoatom org, pos=(0,0,0)\n")
ofs_pml.write("pseudoatom s0, pos=(%f,%f,%f)\n" % tuple(s0 * 100))
ofs_pml.write("as spheres, org s0\n")
ofs_pml.write("color red, s0\n")
ofs_pml.write("distance selection1=org, selection2=s0\n")
ofs_pml.write("set sphere_scale, 10\n")
data = (d_map * 1000).astype(numpy.int32)
cbf.save_numpy_data_as_cbf(data.flatten(), data.shape[1], data.shape[0],
"d_map", "d_map_x1000.cbf")
