def time_it(path, buffer_size): if (buffer_size is None): buffer_size = streambuf.default_buffer_size print("Buffer is %i bytes" % buffer_size) path = os.path.expanduser(path) input = open(path, 'r') inp_buf = streambuf(python_file_obj=input, buffer_size=buffer_size) ext.time_read(input.name, inp_buf) output = open("tmp_tst_python_streambuf", "w") out_buf = streambuf(python_file_obj=output, buffer_size=buffer_size) ext.time_write(output.name, out_buf)
def _get_endianic_raw_data(self, size): big_endian = self._header_dictionary["BYTE_ORDER"] == "big_endian" with self.open_file(self._image_file, "rb") as fh: fh.seek(self._header_size) if big_endian == is_big_endian(): raw_data = read_uint16(streambuf(fh), int(size[0] * size[1])) else: raw_data = read_uint16_bs(streambuf(fh), int(size[0] * size[1])) # note that x and y are reversed here raw_data.reshape(flex.grid(size[1], size[0])) return raw_data
def channel_pixels(ROI,wavelength_A,flux,N,UMAT_nm,Amatrix_rot,fmodel_generator,output): energy_dependent_fmodel=False if energy_dependent_fmodel: fmodel_generator.reset_wavelength(wavelength_A) fmodel_generator.reset_specific_at_wavelength( label_has="FE1",tables=Fe_oxidized_model,newvalue=wavelength_A) fmodel_generator.reset_specific_at_wavelength( label_has="FE2",tables=Fe_reduced_model,newvalue=wavelength_A) print("USING scatterer-specific energy-dependent scattering factors") sfall_channel = fmodel_generator.get_amplitudes() elif use_g_sfall: global g_sfall if g_sfall is None: g_sfall = fmodel_generator.get_amplitudes() sfall_channel = g_sfall else: sfall_channel = sfall_7122 SIM = nanoBragg(detpixels_slowfast=(3000,3000),pixel_size_mm=0.11,Ncells_abc=(N,N,N), wavelength_A=wavelength_A,verbose=0) SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.region_of_interest = ROI SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 50 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image SIM.distance_mm=141.7 SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample=1 SIM.wavelength_A = wavelength_A SIM.polarization=1 SIM.default_F=0 SIM.Fhkl=sfall_channel SIM.Amatrix_RUB = Amatrix_rot SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter=False # flux is always in photons/s SIM.flux=flux SIM.exposure_s=1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm=0.003 #cannot make this 3 microns; spots are too intense temp=SIM.Ncells_abc print("Ncells_abc=",SIM.Ncells_abc) SIM.Ncells_abc=temp from libtbx.development.timers import Profiler P = Profiler("nanoBragg") SIM.printout=True fast = ROI[1][0] + (ROI[1][1]-ROI[1][0])//2 slow = ROI[0][0] + (ROI[0][1]-ROI[0][0])//2 SIM.printout_pixel_fastslow=(slow,fast) from boost_adaptbx.boost.python import streambuf # will deposit printout into output as side effect SIM.add_nanoBragg_spots_nks(streambuf(output)) del P print(("SIM count > 0",(SIM.raw_pixels>0).count(True))) return SIM
def _read_cbf_image(self): start_tag = binascii.unhexlify("0c1a04d5") with self.open_file(self._image_file, "rb") as fh: data = fh.read() data_offset = data.find(start_tag) + 4 cbf_header = self._parse_cbf_header(data[:data_offset - 4].decode( "ascii", "ignore")) if cbf_header["byte_offset"]: pixel_values = uncompress( packed=data[data_offset:data_offset + cbf_header["size"]], fast=cbf_header["fast"], slow=cbf_header["slow"], ) elif cbf_header["no_compression"]: assert len(self.get_detector()) == 1 with self.open_file(self._image_file) as f: f.read(data_offset) pixel_values = read_int32(streambuf(f), cbf_header["length"]) pixel_values.reshape( flex.grid(cbf_header["slow"], cbf_header["fast"])) else: raise ValueError( "Compression of type other than byte_offset or none is not supported (contact authors)" ) return pixel_values
def _add_nanoBragg_spots(self): if self.using_diffBragg_spots: self.D.add_diffBragg_spots() else: rois = self.rois if rois is None: rois = [self._full_roi] _rawpix = None # cuda_add_spots doesnt add spots, it resets each time.. hence we need this for roi in rois: if len(roi)==4: roi = (roi[0], roi[1]), (roi[2],roi[3]) self.D.region_of_interest = roi if self.using_cuda: self.D.add_nanoBragg_spots_cuda() if _rawpix is None and len(rois) > 1: _rawpix = deepcopy(self.D.raw_pixels) elif _rawpix is not None: _rawpix += self.D.raw_pixels elif self.using_omp: from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect from six.moves import StringIO self.D.progress_meter = False self.D.add_nanoBragg_spots_nks(streambuf(StringIO())) else: self.D.add_nanoBragg_spots() if self.using_cuda and _rawpix is not None: self.D.raw_pixels = _rawpix
def exercise_partial_read(self): self.create_file_object(mode='r') words = ext.test_read(streambuf(self.file_object), "partial read") assert words == "Coding, should, " trailing = self.file_object.read() assert trailing == " be fun" self.file_object.close()
def channel_pixels(wavelength_A, flux, N, UMAT_nm, Amatrix_rot, fmodel_generator, local_data): fmodel_generator.reset_wavelength(wavelength_A) fmodel_generator.reset_specific_at_wavelength( label_has="FE1", tables=local_data.get("Fe_oxidized_model"), newvalue=wavelength_A) fmodel_generator.reset_specific_at_wavelength( label_has="FE2", tables=local_data.get("Fe_reduced_model"), newvalue=wavelength_A) print("USING scatterer-specific energy-dependent scattering factors") sfall_channel = fmodel_generator.get_amplitudes() SIM = nanoBragg(detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image SIM.distance_mm = 141.7 SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 SIM.default_F = 0 SIM.Fhkl = sfall_channel SIM.Amatrix_RUB = Amatrix_rot SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter = False # flux is always in photons/s SIM.flux = flux SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp from libtbx.development.timers import Profiler P = Profiler("nanoBragg") if add_spots_algorithm == "NKS": from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect SIM.add_nanoBragg_spots_nks(streambuf(StringIO())) elif add_spots_algorithm == "JH": SIM.add_nanoBragg_spots() elif add_spots_algorithm == "cuda": SIM.add_nanoBragg_spots_cuda() else: raise Exception("unknown spots algorithm") del P return SIM
def exercise_seek_and_read(self): self.create_instrumented_file_object(mode='r') words = ext.test_read(streambuf(self.file_object), "read and seek") assert words == "should, should, uld, ding, fun, [ eof ]" n = streambuf.default_buffer_size soughts = self.file_object.seek_call_log # stringent tests carefully crafted to make sure the seek-in-buffer # optimisation works as expected # C.f. the comment in the C++ function actual_write_test assert soughts[-1] == (-4,2) soughts = soughts[:-1] if n >= 14: assert soughts == [] else: assert soughts[0] == (6,0) soughts = soughts[1:] if 8 <= n <= 13: assert len(soughts) == 1 and self.only_seek_cur(soughts) elif n == 7: assert len(soughts) == 2 and self.only_seek_cur(soughts) else: assert soughts[0] == (6,0) soughts = soughts[1:] assert self.only_seek_cur(soughts) if n == 4: assert len(soughts) == 1 else: assert len(soughts) == 2 self.file_object.close()
def to_smv_format_py(self,fileout,intfile_scale=0.0,debug_x=-1,debug_y=-1, rotmat=False,extra=None,verbose=False,gz=False): byte_order = "little_endian"; #recast the image file write to Python to afford extra options: rotmat, extra, gz if gz: from libtbx.smart_open import for_writing outfile = for_writing(file_name=fileout+".gz", gzip_mode="wb") else: outfile = open(fileout,"wb"); outfile.write(("{\nHEADER_BYTES=1024;\nDIM=2;\nBYTE_ORDER=%s;\nTYPE=unsigned_short;\n"%byte_order).encode()); outfile.write(b"SIZE1=%d;\nSIZE2=%d;\nPIXEL_SIZE=%g;\nDISTANCE=%g;\n"%( self.detpixels_fastslow[0],self.detpixels_fastslow[1],self.pixel_size_mm,self.distance_mm)); outfile.write(b"WAVELENGTH=%g;\n"%self.wavelength_A); outfile.write(b"BEAM_CENTER_X=%g;\nBEAM_CENTER_Y=%g;\n"%self.beam_center_mm); outfile.write(b"ADXV_CENTER_X=%g;\nADXV_CENTER_Y=%g;\n"%self.adxv_beam_center_mm); outfile.write(b"MOSFLM_CENTER_X=%g;\nMOSFLM_CENTER_Y=%g;\n"%self.mosflm_beam_center_mm); outfile.write(b"DENZO_X_BEAM=%g;\nDENZO_Y_BEAM=%g;\n"%self.denzo_beam_center_mm); outfile.write(b"DIALS_ORIGIN=%g,%g,%g\n"%self.dials_origin_mm); outfile.write(b"XDS_ORGX=%g;\nXDS_ORGY=%g;\n"%self.XDS_ORGXY); outfile.write(b"CLOSE_DISTANCE=%g;\n"%self.close_distance_mm); outfile.write(b"PHI=%g;\nOSC_START=%g;\nOSC_RANGE=%g;\n"%(self.phi_deg,self.phi_deg,self.osc_deg)); outfile.write(b"TIME=%g;\n"%self.exposure_s); outfile.write(b"TWOTHETA=%g;\n"%self.detector_twotheta_deg); outfile.write(b"DETECTOR_SN=000;\n"); outfile.write(b"ADC_OFFSET=%g;\n"%self.adc_offset_adu); outfile.write(b"BEAMLINE=fake;\n"); if rotmat: from scitbx.matrix import sqr RSABC = sqr(self.Amatrix).inverse().transpose() outfile.write( ("DIRECT_SPACE_ABC=%s;\n"%(",".join([repr(a) for a in RSABC.elems]))).encode() ) if extra is not None: outfile.write(extra.encode()) outfile.write(b"}\f"); assert outfile.tell() < 1024, "SMV header too long, please edit this code and ask for more bytes." while ( outfile.tell() < 1024 ): outfile.write(b" ") from six import PY3 if PY3: # Python3-compatible method for populating the output buffer. # Py2 implementation is more elegant in that the streambuf may be passed to C++, # and the data are gzipped in chunks (default 1024). Py3 will not accept this method # as it is PyString-based, with no converter mechanisms to bring data into PyBytes. # The Py3 method brings the full data in one chunk into PyBytes and then populates # the output buffer in Python rather than C++. image_bytes = self.raw_pixels_unsigned_short_as_python_bytes(intfile_scale,debug_x,debug_y) ptr = 0; nbytes = len(image_bytes) while (ptr < nbytes): # chunked output necessary to prevent intermittent MemoryError outfile.write(image_bytes[ptr : min(ptr + 65536, nbytes)]) ptr += 65536 outfile.close(); return from boost_adaptbx.boost.python import streambuf self.to_smv_format_streambuf(streambuf(outfile),intfile_scale,debug_x,debug_y) outfile.close();
def channel_pixels(wavelength_A, flux, N, UMAT_nm, Amatrix_rot, rank, sfall_channel): if rank in [0, 7]: print("USING scatterer-specific energy-dependent scattering factors") SIM = nanoBragg(detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image SIM.distance_mm = 141.7 SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 SIM.default_F = 0 SIM.Fhkl = sfall_channel SIM.Amatrix_RUB = Amatrix_rot SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter = False # flux is always in photons/s SIM.flux = flux SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc if rank in [0, 7]: print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp if rank in [0, 7]: P = Profiler("nanoBragg C++ rank %d" % (rank)) if add_spots_algorithm == "NKS": from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect SIM.add_nanoBragg_spots_nks(streambuf(StringIO())) elif add_spots_algorithm == "JH": SIM.add_nanoBragg_spots() elif add_spots_algorithm == "cuda": SIM.xtal_shape = shapetype.Gauss_argchk devices_per_node = int(os.environ["DEVICES_PER_NODE"]) SIM.device_Id = rank % devices_per_node #if rank==7: # os.system("nvidia-smi") SIM.add_nanoBragg_spots_cuda() else: raise Exception("unknown spots algorithm") if rank in [0, 7]: del P return SIM
def __call__(self, N, UMAT_nm, Amatrix_rot, sfall_channel): SIM = nanoBragg(detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), wavelength_A=self.wavlen[0], verbose=0) SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.mosaic_domains = n_mosaic_domains # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image SIM.mosaic_spread_deg = mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.distance_mm = distance_mm SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.polarization = 1 SIM.default_F = 0 SIM.Fhkl = sfall_channel SIM.Amatrix_RUB = Amatrix_rot SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter = False # flux is always in photons/s #SIM.flux=0.8E10 # number of photons per 100 ps integrated pulse, 24-bunch APS SIM.flux = 0.8E12 # number of photons per 100 ps integrated pulse, 24-bunch APS SIM.xray_source_wavelengths_A = self.wavlen SIM.xray_source_intensity_fraction = self.norm_intensity SIM.xray_source_XYZ = self.source_XYZ SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp from libtbx.development.timers import Profiler P = Profiler("nanoBragg") if add_spots_algorithm == "NKS": print("USING NKS") from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect SIM.add_nanoBragg_spots_nks(streambuf(StringIO())) elif add_spots_algorithm == "JH": print("USING JH") SIM.add_nanoBragg_spots() elif add_spots_algorithm == "cuda": print("USING cuda") SIM.add_nanoBragg_spots_cuda() else: raise Exception("unknown spots algorithm") del P return SIM
def exercise_read_failure(self): self.create_file_object(mode='r') self.file_object.close() try: ext.test_read(streambuf(self.file_object), "read") except ValueError: pass else: raise Exception_expected self.file_object.close()
def exercise_write_failure(self): import platform if (platform.platform().find("redhat-8.0") >= 0): return # avoid Abort self.create_file_object(mode='r') try: ext.test_write(streambuf(self.file_object), "write") except IOError as err: pass else: raise Exception_expected self.file_object.close()
def get_raw_data(self): """Read the data - assuming it is streaming 4-byte unsigned ints following the header...""" assert len(self.get_detector()) == 1 size = self.get_detector()[0].get_image_size() f = self.open_file(self._image_file) f.read(int(self._header_dictionary["HEADER_BYTES"])) raw_data = read_int32(streambuf(f), int(size[0] * size[1])) raw_data.reshape(flex.grid(size[1], size[0])) return raw_data
def _read_raw_data(self, f): """Read raw data from a positioned file""" # is this image a type we can read? assert self._data_type in ["h", "f", "B", "l", "b", "H", "I", "d"] if self._data_type == "f": raw_data = read_float32(streambuf(f), self._image_num_elements) elif self._data_type == "B": raw_data = read_uint8(streambuf(f), self._image_num_elements) elif self._data_type == "h": raw_data = read_int16(streambuf(f), self._image_num_elements) elif self._data_type == "H": raw_data = read_uint16(streambuf(f), self._image_num_elements) elif self._data_type == "l": raw_data = read_int32(streambuf(f), self._image_num_elements) elif self._data_type == "I": raw_data = read_uint32(streambuf(f), self._image_num_elements) # no C++ reader for remaining types (should be unusual anyway) else: vals = struct.unpack( self._byteord + self._data_type * self._image_num_elements, f.read(self._data_size * self._image_num_elements), ) if self._data_type == "d": raw_data = flex.double(vals) if self._data_type in ["b", "?"]: raw_data = flex.int(vals) raw_data.reshape(flex.grid(self._image_size[1], self._image_size[0])) return raw_data
def get_raw_data(self): """Get the pixel intensities (i.e. read the image and return as a flex array of integers.)""" # currently have no non-little-endian machines... assert len(self.get_detector()) == 1 image_size = self.get_detector()[0].get_image_size() with self.open_file(self._image_file) as fh: fh.seek(self._header_size) raw_data = read_uint16(streambuf(fh), int(image_size[0] * image_size[1])) raw_data.reshape(flex.grid(image_size[1], image_size[0])) return raw_data
def perform_one_simulation_optimized(self,model,ROI,models4): # models4 is a dict of direct_matrices with keys "Amat","Amat_dx", "Amat_dy", "Amat_dz" Amatrix_rot = models4[model] self.SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter Amatrecover = sqr(self.SIM.Amatrix).transpose() # recovered Amatrix from SIM Ori = crystal_orientation.crystal_orientation(Amatrecover, crystal_orientation.basis_type.reciprocal) self.SIM.raw_pixels.fill(0.0) # effectively initializes the data pixels for a new simulation temp_fill_zeroes_for_roi = self.SIM.raw_pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]] channel_summation_roi_only = self.SIM.raw_pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]] self.SIM.seed = 1 # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) output = StringIO() # open("myfile","w") prefix = "key_slow_nonoise" from LS49.work2_for_aca_lsq.util_partiality import response_plot make_response_plot = response_plot(enable=False,title=prefix) self.SIM.region_of_interest = ROI from boost_adaptbx.boost.python import streambuf self.SIM.printout=True # only for the purpose of getting the P1 Miller index and structure factor fast = ROI[1][0] + (ROI[1][1]-ROI[1][0])//2 slow = ROI[0][0] + (ROI[0][1]-ROI[0][0])//2 self.SIM.printout_pixel_fastslow=(slow,fast) for x in range(0,100,2): #len(flux)): if self.flux[x]==0.0:continue # initialize ROI-only to zero self.SIM.raw_pixels.matrix_paste_block_in_place(temp_fill_zeroes_for_roi,ROI[1][0],ROI[0][0]) self.SIM.wavelength_A = self.wavlen[x] self.SIM.flux = self.flux[x] self.SIM.add_nanoBragg_spots_nks(streambuf(output)) self.SIM.printout = False # only do the printout once through incremental_signal = self.SIM.raw_pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]] * self.crystal.domains_per_crystal make_response_plot.append_channel_full_region(x,incremental_signal) if x in [26,40,54,68]: # subsample 7096, 7110, 7124, 7138 eV #print ("----------------------> subsample", x) make_response_plot.incr_subsample_full_region(x,incremental_signal) make_response_plot.increment_full_region(x,incremental_signal) channel_summation_roi_only += incremental_signal; message = output.getvalue().split() miller = (int(message[4]),int(message[5]),int(message[6])) # nanoBragg P1 index intensity = float(message[9]); make_response_plot.plot(channel_summation_roi_only,miller) return dict(roi_pixels=channel_summation_roi_only,miller=miller,intensity=intensity, channels=make_response_plot.channels)
def _get_raw_data(self, index): data_offset = self._header_dictionary["DataOffsetArray"][index] with FormatSER.open_file(self._image_file, "rb") as f: f.seek(data_offset) d = {} d["CalibrationOffsetX"] = struct.unpack("<d", f.read(8))[0] d["CalibrationDeltaX"] = struct.unpack("<d", f.read(8))[0] d["CalibrationElementX"] = struct.unpack("<I", f.read(4))[0] d["CalibrationOffsetY"] = struct.unpack("<d", f.read(8))[0] d["CalibrationDeltaY"] = struct.unpack("<d", f.read(8))[0] d["CalibrationElementY"] = struct.unpack("<I", f.read(4))[0] d["DataType"] = struct.unpack("<H", f.read(2))[0] if d["DataType"] == 6: read_pixel = dxtbx.ext.read_int32 elif d["DataType"] == 5: read_pixel = dxtbx.ext.read_int16 elif d["DataType"] == 3: read_pixel = dxtbx.ext.read_uint32 elif d["DataType"] == 2: read_pixel = dxtbx.ext.read_uint16 elif d["DataType"] == 1: read_pixel = dxtbx.ext.read_uint8 else: raise RuntimeError( "Image {} data is of an unsupported type".format(index + 1) ) d["ArraySizeX"] = struct.unpack("<I", f.read(4))[0] d["ArraySizeY"] = struct.unpack("<I", f.read(4))[0] nelts = d["ArraySizeX"] * d["ArraySizeY"] raw_data = read_pixel(streambuf(f), nelts) # Check image size is as expected (same as the first image) if d["ArraySizeX"] != self._header_dictionary["ArraySizeX"]: raise RuntimeError( "Image {} has an unexpected array size in X".format(index + 1) ) if d["ArraySizeY"] != self._header_dictionary["ArraySizeY"]: raise RuntimeError( "Image {} has an unexpected array size in Y".format(index + 1) ) image_size = (d["ArraySizeX"], d["ArraySizeY"]) raw_data.reshape(flex.grid(image_size[1], image_size[0])) return raw_data
def get_raw_data(self): """Get the pixel intensities (i.e. read the image and return as a flex array of integers.)""" # currently have no non-little-endian machines... assert self._tiff_byte_order == FormatTIFF.LITTLE_ENDIAN bias = int(round(self._get_rayonix_bias())) assert len(self.get_detector()) == 1 size = self.get_detector()[0].get_image_size() f = FormatTIFF.open_file(self._image_file) f.read(self._header_size) raw_data = read_uint16(streambuf(f), int(size[0] * size[1])) - bias image_size = self.get_detector()[0].get_image_size() raw_data.reshape(flex.grid(image_size[1], image_size[0])) return raw_data
def add_channel_pixels(self, wavelength_A, flux, rank, algo="cuda"): """ :param wavelength_A: :param flux: :param rank: :param algo: :return: """ self.SIM.flux = flux self.SIM.wavelength_A = wavelength_A P = Profiler("nanoBragg C++ rank %d" % (rank)) if algo == "NKS": self.SIM.add_nanoBragg_spots_nks(streambuf(StringIO())) self.raw_pixels = self.SIM.raw_pixels # NOTE: this actually incremenents hack for now, because cuda doesnt add spots elif algo == "JH": self.SIM.add_nanoBragg_spots() self.raw_pixels = self.SIM.raw_pixels elif algo == "cuda": self.SIM.add_nanoBragg_spots_cuda() #self.SIM.add_nanoBragg_spots_cuda_light() # FIXME: this should be updated to not initialize self.raw_pixels += self.SIM.raw_pixels # NOTE: will be on GPU else: raise Exception("unknown spots algorithm '%s' " % algo)
def get_raw_data(self, index): # is this image a type we can read? assert self._data_type in ["h", "f", "B", "l", "b", "H", "I", "d"] with FormatGatanDM4.open_file(self._image_file, "rb") as f: f.seek(self._data_offset) skip_bytes = index * self._image_num_elements * self._data_size f.seek(skip_bytes, whence=1) if self._data_type == "f": raw_data = read_float32(streambuf(f), self._image_num_elements) elif self._data_type == "B": raw_data = read_uint8(streambuf(f), self._image_num_elements) elif self._data_type == "h": raw_data = read_int16(streambuf(f), self._image_num_elements) elif self._data_type == "H": raw_data = read_uint16(streambuf(f), self._image_num_elements) elif self._data_type == "l": raw_data = read_int32(streambuf(f), self._image_num_elements) elif self._data_type == "I": raw_data = read_uint32(streambuf(f), self._image_num_elements) # no C++ reader for remaining types (should be unusual anyway) else: vals = struct.unpack( self._byteord + self._data_type * self._image_num_elements, f.read(self._data_size * self._image_num_elements), ) if self._data_type == "d": raw_data = flex.double(vals) if self._data_type in ["b", "?"]: raw_data = flex.int(vals) raw_data.reshape(flex.grid(self._image_size[1], self._image_size[0])) return raw_data
def read_from_stdin(): written = ext.test_read(streambuf(sys.stdin), "read") assert written == "Veni, Vidi, Vici, [ fail, eof ]", written
def exercise_read(self): self.create_file_object(mode='r') words = ext.test_read(streambuf(self.file_object), "read") assert words == "Coding, should, be, fun, [ fail, eof ]" self.file_object.close()
def get_raw_data(self): """Get the pixel intensities (i.e. read the image and return as a flex array of integers.)""" # It is better to catch FORMAT 86 here and fail with a sensible error msg # as soon as something is attempted with the image data rather than in # the understand method. Otherwise the user gets FormatBruker reading the # image improperly but without failing if self.header_dict["FORMAT"] != "100": raise NotImplementedError( "Only FORMAT 100 images from the Photon II are currently supported" ) f = self.open_file(self._image_file, "rb") header_size = int(self.header_dict["HDRBLKS"]) * 512 f.read(header_size) if is_big_endian(): read_2b = read_uint16_bs read_4b = read_uint32_bs else: read_2b = read_uint16 read_4b = read_uint32 # NPIXELB stores the number of bytes/pixel for the data and the underflow # table. We expect 1 byte for underflows and either 2 or 1 byte per pixel # for the data npixelb = [int(e) for e in self.header_dict["NPIXELB"].split()] assert npixelb[1] == 1 if npixelb[0] == 1: read_data = read_uint8 elif npixelb[0] == 2: read_data = read_2b else: raise IncorrectFormatError( "{} bytes per pixel is not supported".format(npixelb[0])) nrows = int(self.header_dict["NROWS"].split()[0]) ncols = int(self.header_dict["NCOLS"].split()[0]) raw_data = read_data(streambuf(f), nrows * ncols) image_size = (nrows, ncols) raw_data.reshape(flex.grid(*image_size)) (num_underflows, num_2b_overflows, num_4b_overflows) = [ int(e) for e in self.header_dict["NOVERFL"].split() ] # read underflows if num_underflows > 0: # stored values are padded to a multiple of 16 bytes nbytes = num_underflows + 15 & ~(15) underflow_vals = read_uint8(streambuf(f), nbytes)[:num_underflows] else: underflow_vals = None # handle 2 byte overflows if num_2b_overflows > 0: # stored values are padded to a multiple of 16 bytes nbytes = num_2b_overflows * 2 + 15 & ~(15) overflow_vals = read_2b(streambuf(f), nbytes // 2)[:num_2b_overflows] overflow = flex.int(nrows * ncols, 0) sel = (raw_data == 255).as_1d() overflow.set_selected(sel, overflow_vals - 255) overflow.reshape(flex.grid(*image_size)) raw_data += overflow # handle 4 byte overflows if num_4b_overflows > 0: # stored values are padded to a multiple of 16 bytes nbytes = num_4b_overflows * 4 + 15 & ~(15) overflow_vals = read_4b(streambuf(f), nbytes // 4)[:num_4b_overflows] overflow = flex.int(nrows * ncols, 0) sel = (raw_data == 65535).as_1d() overflow.set_selected(sel, overflow_vals - 65535) overflow.reshape(flex.grid(*image_size)) raw_data += overflow # handle underflows if underflow_vals is not None: sel = (raw_data == 0).as_1d() underflow = flex.int(nrows * ncols, 0) underflow.set_selected(sel, underflow_vals) underflow.reshape(flex.grid(*image_size)) raw_data += underflow # handle baseline. num_underflows == -1 means no baseline subtraction. See # https://github.com/cctbx/cctbx_project/files/1262952/BISFrameFileFormats.zip if num_underflows != -1: num_exposures = [int(e) for e in self.header_dict["NEXP"].split()] baseline = num_exposures[2] raw_data += baseline return raw_data