def run(infile): h5 = h5py.File(infile, "r") if h5["/entry/instrument/detector/flatfield_correction_applied"].value: print "Correction was already applied." return ff = h5["/entry/instrument/detector/detectorSpecific/flatfield"][:] outdir = "flatfield_applied_%s" % os.path.splitext(os.path.basename(infile))[0] if not os.path.exists(outdir): os.mkdir(outdir) cbf.save_numpy_data_as_cbf((ff*1000).astype(numpy.uint32).flatten(), size1=ff.shape[1], size2=ff.shape[0], title="flatfield_x1000", cbfout=os.path.join(outdir, "flatfield.cbf")) for k in sorted(h5["/entry/data"].keys()): outfile = os.path.join(outdir, h5["/entry/data"].get(k, getlink=True).filename) ds = h5["/entry/data"][k] h5o = h5py.File(outfile, "w") h5o.create_group("/entry") h5o["/entry"].attrs["NX_class"] = "NXentry" h5o.create_group("/entry/data") h5o["/entry/data"].attrs["NX_class"] = "NXdata" dataset = h5o.create_dataset("/entry/data/data", ds.shape, compression=bitshuffle.h5.H5FILTER, compression_opts=(0, bitshuffle.h5.H5_COMPRESS_LZ4), chunks=ds.chunks, dtype=ds.dtype) dataset.attrs["image_nr_low"] = ds.attrs["image_nr_low"] dataset.attrs["image_nr_high"] = ds.attrs["image_nr_high"] # Don't want to load all data to memory at the same time for i in xrange(ds.shape[0]): print "processing %6d/%d in %s" % (i+1, ds.shape[0], k) dataset[i,] = ds[i,] * ff + .5
def run(mrcin, prefix=None): f = mrcfile.open(mrcin) if prefix is None: prefix = os.path.splitext(os.path.basename(mrcin))[0] data = f.data if data.ndim == 2: data = data.reshape(-1, *data.shape) print "MRC file loaded" print " shape = %s" % (data.shape,) print " dtype = %s" % data.dtype print for i in xrange(data.shape[0]): size2, size1 = data[i].shape # XXX really? cbfout = "%s_%.3d.cbf"%(prefix, i+1) print "Writing %s" % cbfout cbf.save_numpy_data_as_cbf(data[i].flatten(), size1, size2, "%s:%d"%(mrcin, i+1), cbfout, pilatus_header=""" # Detector: %(detname)s # Pixel_size %(pixelsize)e m x %(pixelsize)e m # Wavelength %(wavelength)f A # Detector_distance %(distance)f m # Beam_xy (%(beamx)f, %(beamy)f) pixels # Start_angle %(start_angle)%.4f deg. # Angle_increment %(angle_inc).4f deg. """ % dict(detname="????", pixelsize=75.e-6, wavelength=1, distance=300.e-3, beamx=size1/2., beamy=size2/2., start_angle=0, angle_inc=0))
def run(params): xmin, xmax, nxbin, positions = read_correct_lp(params.correct_lp) xstep = (xmax-xmin)/nxbin absorp = read_absorp_cbf(params.absorp_cbf) assert numpy.all(absorp > 13) # if det = read_bkgpix_cbf(params.bkgpix_cbf, positions) datout = open("%s.dat"%params.output.prefix, "w") datout.write("ix xmin xmax ipos posx posy fac\n") for ix in xrange(nxbin): x1, x2 = xmin + ix*xstep, xmin + (ix+1)*xstep tmp = numpy.array(det) for i in xrange(len(positions)): tmp[tmp==i+1] = absorp[i,ix] datout.write("%2d %.2f %.2f %2d %d %d %5d\n" % (ix, x1, x2, i, positions[i][0], positions[i][1], absorp[i,ix])) tmp = numpy.array(tmp, dtype=numpy.int32) cbf.save_numpy_data_as_cbf(tmp.flatten(), tmp.shape[1], tmp.shape[0], "absorp_%.1f-%.1f"% (x1,x2), "%s_%.3d.cbf" % (params.output.prefix, ix)) datout.close()
def save_cbf(wavelen, data, cbfout): header = """\ # Detector: NOT PILATUS but MPCCD # Wavelength %f A """ % wavelen height, width = data.shape #print "Saving", cbfout, height, width, data cbf.save_numpy_data_as_cbf(data.reshape(width*height), width, height, "hdf5_converted", str(cbfout), pilatus_header=header)
def run(cbfin, cbfout, repl=None): data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(cbfin) max_I = max(data) if repl is None: repl = max_I+100 data[data<0] = repl cbf.save_numpy_data_as_cbf(data, ndimfast, ndimmid, "negative_replaced", cbfout) print "Launch:" print "adxv -overload %d %s" % (max_I, cbfout)
def run(cbfin, cbfout, repl=None): data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(cbfin) max_I = max(data) if repl is None: repl = max_I + 100 data[data < 0] = repl cbf.save_numpy_data_as_cbf(data, ndimfast, ndimmid, "negative_replaced", cbfout) print "Launch:" print "adxv -overload %d %s" % (max_I, cbfout)
def run(files, cbfout): merged = None for i, f in enumerate(files): repl = -10 * (i+1) print "%s %d" % (f, repl) data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(f) if i == 0: merged = data.copy() continue merged[data==-10] = 65540 # repl # for adxv visualization. only assuming two files. cbf.save_numpy_data_as_cbf(merged, ndimfast, ndimmid, "merged_predictions", cbfout)
def extract_to_minicbf(h5master, frameno_or_path, cbfout, binning=1): from yamtbx.dataproc import cbf from yamtbx.dataproc.XIO.plugins import eiger_hdf5_interpreter if type(frameno_or_path) in (tuple, list): data = extract_data_range_sum(h5master, frameno_or_path) nframes = len(frameno_or_path) elif type(frameno_or_path) is int: data = extract_data(h5master, frameno_or_path) nframes = 1 else: data = extract_data_path(h5master, frameno_or_path) nframes = 1 if data is None: raise RuntimeError("Cannot extract frame %s from %s" % (frameno_or_path, h5master)) h = eiger_hdf5_interpreter.Interpreter().getRawHeadDict(h5master) h5 = h5py.File(h5master, "r") if binning > 1: beamxy = h["BeamX"], h["BeamY"] data, (h["BeamX"], h["BeamY"]) = software_binning(data, binning, beamxy) h["Detector"] = "Unknown" if "/entry/instrument/detector/description" in h: # EIGER2 does not have this field? h["Detector"] = h5["/entry/instrument/detector/description"].value h["ExposurePeriod"] = h5["/entry/instrument/detector/frame_time"].value h["PhiWidth"] *= nframes cbf.save_numpy_data_as_cbf(data.flatten(), size1=data.shape[1], size2=data.shape[0], title="", cbfout=cbfout, pilatus_header="""\ # Detector: %(Detector)s, S/N %(SerialNumber)s # Pixel_size %(PixelX)e m x %(PixelY)e m # %(SensorMaterial)s sensor, thickness %(SensorThickness).3e m # Exposure_time %(ExposureTime).6f s # Exposure_period %(ExposurePeriod).6f s # Count_cutoff %(Overload)d counts # Wavelength %(Wavelength).6f A # Detector_distance %(Distance).3e m # Beam_xy (%(BeamX).1f, %(BeamY).1f) pixels # Start_angle %(PhiStart).6f deg. # Angle_increment %(PhiWidth).6f deg. """ % h)
def save_cbf(wavelen, data, cbfout): header = """\ # Detector: NOT PILATUS but MPCCD # Wavelength %f A """ % wavelen height, width = data.shape #print "Saving", cbfout, height, width, data cbf.save_numpy_data_as_cbf(data.reshape(width * height), width, height, "hdf5_converted", str(cbfout), pilatus_header=header)
def run(files, cbfout): merged = None for i, f in enumerate(files): repl = -10 * (i + 1) print "%s %d" % (f, repl) data, ndimfast, ndimmid = cbf.load_minicbf_as_numpy(f) if i == 0: merged = data.copy() continue merged[ data == -10] = 65540 # repl # for adxv visualization. only assuming two files. cbf.save_numpy_data_as_cbf(merged, ndimfast, ndimmid, "merged_predictions", cbfout)
def write_minicbf(data, metadata, prefix, frame_range, alpha_step, time_step, params): detector = str(metadata[0]["BinaryResult"]["Detector"]) voltage = float(metadata[0]["Optics"]["AccelerationVoltage"]) wavelen = electron_voltage_to_wavelength(voltage) #width = int(metadata[0]["BinaryResult"]["ImageSize"]["width"]) #/ int(metadata[0]["Detectors"]["ImageSize"]["width"]) #height = int(metadata[0]["BinaryResult"]["ImageSize"]["height"]) # / binning!! width, height = data.shape[:2] # the other way around? binning = int(metadata[0]["BinaryResult"]["ImageSize"]["width"])//width exp_time = float(metadata[0]["Detectors"]["Detector-0"]["ExposureTime"]) px = params.pixel_size*1.e-6 * binning clen = params.clen if params.beamxy is None: beamx, beamy = (width/2., height/2.) else: beamx, beamy = params.beamxy print "There are %d frames" % data.shape[2] for i in xrange(data.shape[2]): if i < frame_range[0] or i > frame_range[1]: print "Ignoring frame %d" % (i+1) continue print "Converting", i+1 frame_date = datetime.datetime.fromtimestamp(int(metadata[i]["CustomProperties"]["Detectors[%s].TimeStamp"%params.detector]["value"])/1.e6) datestr = datetime.datetime.strftime(frame_date, "%Y-%m%-dT%H:%M:%S.%f") start_angle = str(metadata[i]["Stage"]["AlphaTilt"]) cbf.save_numpy_data_as_cbf(data[:,:,i].flatten().astype(numpy.int32)+params.offset, data.shape[0], data.shape[1], "", "%s_%.6d.cbf"%(prefix, i+1), pilatus_header=""" # Detector: %(detector)s # %(datestr)s # Pixel_size %(px).1e m x %(px).1e m # Exposure_time %(exp_time)s s # Exposure_period %(time_step)f s # Wavelength %(wavelen)f A # Detector_distance %(clen)f m # Beam_xy (%(beamx).2f, %(beamy).2f) pixels # Start_angle %(start_angle)s deg. # Angle_increment %(alpha_step)f deg. """ % locals(), header_convention="PILATUS_1.2")
def run(infile): h5 = h5py.File(infile, "r") if h5["/entry/instrument/detector/flatfield_correction_applied"].value: print "Correction was already applied." return ff = h5["/entry/instrument/detector/detectorSpecific/flatfield"][:] outdir = "flatfield_applied_%s" % os.path.splitext( os.path.basename(infile))[0] if not os.path.exists(outdir): os.mkdir(outdir) cbf.save_numpy_data_as_cbf((ff * 1000).astype(numpy.uint32).flatten(), size1=ff.shape[1], size2=ff.shape[0], title="flatfield_x1000", cbfout=os.path.join(outdir, "flatfield.cbf")) for k in sorted(h5["/entry/data"].keys()): outfile = os.path.join(outdir, h5["/entry/data"].get(k, getlink=True).filename) ds = h5["/entry/data"][k] h5o = h5py.File(outfile, "w") h5o.create_group("/entry") h5o["/entry"].attrs["NX_class"] = "NXentry" h5o.create_group("/entry/data") h5o["/entry/data"].attrs["NX_class"] = "NXdata" dataset = h5o.create_dataset( "/entry/data/data", ds.shape, compression=bitshuffle.h5.H5FILTER, compression_opts=(0, bitshuffle.h5.H5_COMPRESS_LZ4), chunks=ds.chunks, dtype=ds.dtype) dataset.attrs["image_nr_low"] = ds.attrs["image_nr_low"] dataset.attrs["image_nr_high"] = ds.attrs["image_nr_high"] # Don't want to load all data to memory at the same time for i in xrange(ds.shape[0]): print "processing %6d/%d in %s" % (i + 1, ds.shape[0], k) dataset[i, ] = ds[i, ] * ff + .5
def extract_to_minicbf(h5master, frameno, cbfout, binning=1): from yamtbx.dataproc import cbf from yamtbx.dataproc.XIO.plugins import eiger_hdf5_interpreter if type(frameno) in (tuple, list): data = extract_data_range_sum(h5master, frameno) nframes = len(frameno) else: data = extract_data(h5master, frameno) nframes = 1 if data is None: raise RuntimeError("Cannot extract frame %s from %s"%(frameno, h5master)) h = eiger_hdf5_interpreter.Interpreter().getRawHeadDict(h5master) h5 = h5py.File(h5master, "r") if binning>1: beamxy = h["BeamX"], h["BeamY"] data, (h["BeamX"], h["BeamY"]) = software_binning(data, binning, beamxy) h["Detector"] = h5["/entry/instrument/detector/description"].value h["ExposurePeriod"] = h5["/entry/instrument/detector/frame_time"].value h["PhiWidth"] *= nframes cbf.save_numpy_data_as_cbf(data.flatten(), size1=data.shape[1], size2=data.shape[0], title="", cbfout=cbfout, pilatus_header="""\ # Detector: %(Detector)s, S/N %(SerialNumber)s # Pixel_size %(PixelX)e m x %(PixelY)e m # %(SensorMaterial)s sensor, thickness %(SensorThickness).3e m # Exposure_time %(ExposureTime).6f s # Exposure_period %(ExposurePeriod).6f s # Count_cutoff %(Overload)d counts # Wavelength %(Wavelength).6f A # Detector_distance %(Distance).3e m # Beam_xy (%(BeamX).1f, %(BeamY).1f) pixels # Start_angle %(PhiStart).6f deg. # Angle_increment %(PhiWidth).6f deg. """ % h)
def run(h5in, cbf_prefix): f = h5py.File(h5in, "r") if "instrument" not in f["entry"]: print "Error: This is not master h5 file." return dname = os.path.dirname(cbf_prefix) if dname != "" and not os.path.exists(dname): os.makedirs(dname) print "dir created:", dname # Analyze pixel_mask pixel_mask = numpy.array(f["entry"]["instrument"]["detector"]["detectorSpecific"]["pixel_mask"]) print "No. of unuseful pixels:" for val in set(pixel_mask.flatten()): if val==0: continue print "", get_mask_info(val), (pixel_mask==val).sum() print # Extract and write data data = filter(lambda x: x.startswith("data_"), f["entry"]) count = 0 for key in sorted(data): print "Extracting", key im = f["entry"][key] print " shape=", im.shape print " dtype=", im.dtype nframes, height, width = im.shape for i in xrange(nframes): count += 1 cbfout = "%s_%.6d.cbf" % (cbf_prefix, count) data = im[i,].astype(numpy.int32) data[pixel_mask>0] = -1 cbf.save_numpy_data_as_cbf(data.reshape(width*height), width, height, "hdf5_converted", cbfout, pilatus_header=make_dummy_pilatus_header(f)) print " ", cbfout print
def on_monitor_timer(self, ev): print "monitoring" response = urllib.urlopen("http://%s/monitor/api/%s/images/monitor" % (self.txtEigerHost.GetValue(), self.txtEigerAPIver.GetValue())) data = response.read() data_size = len(data) curlines = self.txtInfo.GetValue().splitlines() if len(curlines) > 20: self.txtInfo.SetValue("\n".join(curlines)) if data_size < 10000: self.txtInfo.SetValue("%s: %s\n%s" % (time.ctime(), data, self.txtInfo.GetValue())) return #import cPickle as pickle #pickle.dump(data, open("monitor.pkl","wb"), -1) #data = pickle.load(open("monitor.pkl")) def request(query, base="detector"): r = urllib.urlopen("http://%s/%s/api/%s/%s"%(self.txtEigerHost.GetValue(), base, self.txtEigerAPIver.GetValue(), query)).read() return json.loads(r) nx = request("config/x_pixels_in_detector")["value"] ny = request("config/y_pixels_in_detector")["value"] if len(data) < 1000: data = self.last_monitor_image if data is None: return else: byte = data_size // (nx*ny) assert byte == 4 or byte == 2 data = numpy.fromstring(data[8:-102], dtype=numpy.int32 if byte==4 else numpy.int16).reshape(ny,nx) if self.last_monitor_image is not None and (data==self.last_monitor_image).all(): return wavelen = request("config/wavelength")["value"] beamx = request("config/beam_center_x")["value"] beamy = request("config/beam_center_y")["value"] distance = request("config/detector_distance") image_number = request("status/monitor_image_number", base="monitor")["value"] self.txtInfo.SetValue("""\ %s: Downloaded (frame: %.6d) Wavelength: %.5f A Detector_distance: %.2f %s Beam_xy: %.2f, %.2f %s"""% (time.ctime(), image_number[1]+1, wavelen, distance["value"], str(distance["unit"]), beamx, beamy, self.txtInfo.GetValue())) from yamtbx.dataproc import cbf tmpdir = "/dev/shm" if os.path.isdir("/dev/shm") else tempfile.gettempdir() imgfile = os.path.join("/dev/shm", "adxvtmp-%s-%s.cbf"%(getpass.getuser(), os.getpid())) xp = request("config/x_pixel_size") yp = request("config/y_pixel_size") cbf.save_numpy_data_as_cbf(data.flatten(), size1=data.shape[1], size2=data.shape[0], title="", cbfout=imgfile, pilatus_header="""\ # Detector: Eiger # Pixel_size %(px).5e %(pxu)s x %(py).5e %(pyu)s # Wavelength %(Wavelength).6f A # Detector_distance %(Distance).4e %(DistanceU)s # Beam_xy (%(BeamX).1f, %(BeamY).1f) pixels """ % dict(px=xp["value"], pxu=str(xp["unit"]), py=yp["value"], pyu=str(yp["unit"]), Wavelength=wavelen, Distance=distance["value"], DistanceU=str(distance["unit"]), BeamX=beamx, BeamY=beamy)) self.adxv.open_image(imgfile, raise_window=self.chkMonRaiseW.GetValue()) self.last_monitor_image = data
def run(xds_inp): xp = XPARM() xp.set_info_from_xdsinp_or_inpstr(xdsinp=xds_inp) d_map = numpy.zeros((xp.ny, xp.nx)) # resolution mapping ed = numpy.zeros((3, 3)) ed[:, 0] = xp.X_axis ed[:, 1] = xp.Y_axis ed[:, 2] = numpy.cross(xp.X_axis, xp.Y_axis) ed /= numpy.linalg.norm(ed, axis=0) orgx, orgy = xp.origin fs = xp.distance qx, qy = xp.qx, xp.qy org = -orgx * qx * ed[:, 0] - orgy * qy * ed[:, 1] + fs * ed[:, 2] wavelen = xp.wavelength s0 = xp.incident_beam / numpy.linalg.norm(xp.incident_beam) / wavelen print "qx,qy=", qx, qy print "s0=", s0 print "lambda=", wavelen print "ORG=", org print "ED=" print ed ofs_pdb = open("detector_pos.pdb", "w") ofs_pml = open("for_pymol.pml", "w") for iseg, seg in enumerate(xp.segments): print("Segment %d" % (iseg + 1)) eds = numpy.zeros((3, 3)) eds[:, 0] = seg.eds_x eds[:, 1] = seg.eds_y eds[:, 2] = numpy.cross(seg.eds_x, seg.eds_y) eds /= numpy.linalg.norm(eds, axis=0) edsl = numpy.dot(ed, eds) ix, iy = numpy.meshgrid(range(seg.x1 - 1, seg.x2), range(seg.y1 - 1, seg.y2)) tmp = numpy.zeros((ix.shape[0], ix.shape[1], 3)) for i in range(3): tmp[:, :, i] = qx * (ix - seg.orgxs) * edsl[i, 0] + qy * ( iy - seg.orgys) * edsl[i, 1] + seg.fs * edsl[i, 2] + org[i] write_atom(ofs_pdb, tmp, iseg) ofs_pml.write("bond resi %d and name C1, resi %d and name C2\n" % (iseg, iseg)) ofs_pml.write("bond resi %d and name C2, resi %d and name C4\n" % (iseg, iseg)) ofs_pml.write("bond resi %d and name C3, resi %d and name C4\n" % (iseg, iseg)) ofs_pml.write("bond resi %d and name C3, resi %d and name C1\n" % (iseg, iseg)) tmpdenom = numpy.linalg.norm(tmp, axis=2) * wavelen for i in range(3): tmp[:, :, i] /= tmpdenom s1 = tmp s = s1 - s0 d_map[iy, ix] = 1. / numpy.linalg.norm(s, axis=2) """ for ix in range(seg.x1-1, seg.x2): for iy in range(seg.y1-1, seg.y2): tmp = (qx*(ix-seg.orgxs)*edsl[0,0]+qy*(iy-seg.orgys)*edsl[0,1]+seg.fs*edsl[0,2]+orgx, qx*(ix-seg.orgxs)*edsl[1,0]+qy*(iy-seg.orgys)*edsl[1,1]+seg.fs*edsl[1,2]+orgy, qx*(ix-seg.orgxs)*edsl[2,0]+qy*(iy-seg.orgys)*edsl[2,1]+seg.fs*edsl[2,2]+fs) s1 = tmp/numpy.linalg.norm(tmp)/wavelen s = s1-s0 d_map[iy,ix] = numpy.linalg.norm(s) """ ofs_pml.write("pseudoatom org, pos=(0,0,0)\n") ofs_pml.write("pseudoatom s0, pos=(%f,%f,%f)\n" % tuple(s0 * 100)) ofs_pml.write("as spheres, org s0\n") ofs_pml.write("color red, s0\n") ofs_pml.write("distance selection1=org, selection2=s0\n") ofs_pml.write("set sphere_scale, 10\n") data = (d_map * 1000).astype(numpy.int32) cbf.save_numpy_data_as_cbf(data.flatten(), data.shape[1], data.shape[0], "d_map", "d_map_x1000.cbf")