def time_it(path, buffer_size):
if (buffer_size is None):
buffer_size = streambuf.default_buffer_size
print("Buffer is %i bytes" % buffer_size)
path = os.path.expanduser(path)
input = open(path, 'r')
inp_buf = streambuf(python_file_obj=input, buffer_size=buffer_size)
ext.time_read(input.name, inp_buf)
output = open("tmp_tst_python_streambuf", "w")
out_buf = streambuf(python_file_obj=output, buffer_size=buffer_size)
ext.time_write(output.name, out_buf)
def _get_endianic_raw_data(self, size):
big_endian = self._header_dictionary["BYTE_ORDER"] == "big_endian"
with self.open_file(self._image_file, "rb") as fh:
fh.seek(self._header_size)
if big_endian == is_big_endian():
raw_data = read_uint16(streambuf(fh), int(size[0] * size[1]))
else:
raw_data = read_uint16_bs(streambuf(fh),
int(size[0] * size[1]))
# note that x and y are reversed here
raw_data.reshape(flex.grid(size[1], size[0]))
return raw_data
def channel_pixels(ROI,wavelength_A,flux,N,UMAT_nm,Amatrix_rot,fmodel_generator,output):
energy_dependent_fmodel=False
if energy_dependent_fmodel:
fmodel_generator.reset_wavelength(wavelength_A)
fmodel_generator.reset_specific_at_wavelength(
label_has="FE1",tables=Fe_oxidized_model,newvalue=wavelength_A)
fmodel_generator.reset_specific_at_wavelength(
label_has="FE2",tables=Fe_reduced_model,newvalue=wavelength_A)
print("USING scatterer-specific energy-dependent scattering factors")
sfall_channel = fmodel_generator.get_amplitudes()
elif use_g_sfall:
global g_sfall
if g_sfall is None: g_sfall = fmodel_generator.get_amplitudes()
sfall_channel = g_sfall
else:
sfall_channel = sfall_7122
SIM = nanoBragg(detpixels_slowfast=(3000,3000),pixel_size_mm=0.11,Ncells_abc=(N,N,N),
wavelength_A=wavelength_A,verbose=0)
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.region_of_interest = ROI
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = 50 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
SIM.distance_mm=141.7
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample=1
SIM.wavelength_A = wavelength_A
SIM.polarization=1
SIM.default_F=0
SIM.Fhkl=sfall_channel
SIM.Amatrix_RUB = Amatrix_rot
SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian
SIM.progress_meter=False
# flux is always in photons/s
SIM.flux=flux
SIM.exposure_s=1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm=0.003 #cannot make this 3 microns; spots are too intense
temp=SIM.Ncells_abc
print("Ncells_abc=",SIM.Ncells_abc)
SIM.Ncells_abc=temp
from libtbx.development.timers import Profiler
P = Profiler("nanoBragg")
SIM.printout=True
fast = ROI[1][0] + (ROI[1][1]-ROI[1][0])//2
slow = ROI[0][0] + (ROI[0][1]-ROI[0][0])//2
SIM.printout_pixel_fastslow=(slow,fast)
from boost_adaptbx.boost.python import streambuf # will deposit printout into output as side effect
SIM.add_nanoBragg_spots_nks(streambuf(output))
del P
print(("SIM count > 0",(SIM.raw_pixels>0).count(True)))
return SIM
def _read_cbf_image(self):
start_tag = binascii.unhexlify("0c1a04d5")
with self.open_file(self._image_file, "rb") as fh:
data = fh.read()
data_offset = data.find(start_tag) + 4
cbf_header = self._parse_cbf_header(data[:data_offset - 4].decode(
"ascii", "ignore"))
if cbf_header["byte_offset"]:
pixel_values = uncompress(
packed=data[data_offset:data_offset + cbf_header["size"]],
fast=cbf_header["fast"],
slow=cbf_header["slow"],
)
elif cbf_header["no_compression"]:
assert len(self.get_detector()) == 1
with self.open_file(self._image_file) as f:
f.read(data_offset)
pixel_values = read_int32(streambuf(f), cbf_header["length"])
pixel_values.reshape(
flex.grid(cbf_header["slow"], cbf_header["fast"]))
else:
raise ValueError(
"Compression of type other than byte_offset or none is not supported (contact authors)"
)
return pixel_values
def _add_nanoBragg_spots(self):
if self.using_diffBragg_spots:
self.D.add_diffBragg_spots()
else:
rois = self.rois
if rois is None:
rois = [self._full_roi]
_rawpix = None # cuda_add_spots doesnt add spots, it resets each time.. hence we need this
for roi in rois:
if len(roi)==4:
roi = (roi[0], roi[1]), (roi[2],roi[3])
self.D.region_of_interest = roi
if self.using_cuda:
self.D.add_nanoBragg_spots_cuda()
if _rawpix is None and len(rois) > 1:
_rawpix = deepcopy(self.D.raw_pixels)
elif _rawpix is not None:
_rawpix += self.D.raw_pixels
elif self.using_omp:
from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect
from six.moves import StringIO
self.D.progress_meter = False
self.D.add_nanoBragg_spots_nks(streambuf(StringIO()))
else:
self.D.add_nanoBragg_spots()
if self.using_cuda and _rawpix is not None:
self.D.raw_pixels = _rawpix
def exercise_partial_read(self): self.create_file_object(mode='r') words = ext.test_read(streambuf(self.file_object), "partial read") assert words == "Coding, should, " trailing = self.file_object.read() assert trailing == " be fun" self.file_object.close()
def channel_pixels(wavelength_A, flux, N, UMAT_nm, Amatrix_rot,
fmodel_generator, local_data):
fmodel_generator.reset_wavelength(wavelength_A)
fmodel_generator.reset_specific_at_wavelength(
label_has="FE1",
tables=local_data.get("Fe_oxidized_model"),
newvalue=wavelength_A)
fmodel_generator.reset_specific_at_wavelength(
label_has="FE2",
tables=local_data.get("Fe_reduced_model"),
newvalue=wavelength_A)
print("USING scatterer-specific energy-dependent scattering factors")
sfall_channel = fmodel_generator.get_amplitudes()
SIM = nanoBragg(detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
wavelength_A=wavelength_A,
verbose=0)
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
SIM.distance_mm = 141.7
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = wavelength_A
SIM.polarization = 1
SIM.default_F = 0
SIM.Fhkl = sfall_channel
SIM.Amatrix_RUB = Amatrix_rot
SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian
SIM.progress_meter = False
# flux is always in photons/s
SIM.flux = flux
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
print("Ncells_abc=", SIM.Ncells_abc)
SIM.Ncells_abc = temp
from libtbx.development.timers import Profiler
P = Profiler("nanoBragg")
if add_spots_algorithm == "NKS":
from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect
SIM.add_nanoBragg_spots_nks(streambuf(StringIO()))
elif add_spots_algorithm == "JH":
SIM.add_nanoBragg_spots()
elif add_spots_algorithm == "cuda":
SIM.add_nanoBragg_spots_cuda()
else:
raise Exception("unknown spots algorithm")
del P
return SIM
def exercise_seek_and_read(self):
self.create_instrumented_file_object(mode='r')
words = ext.test_read(streambuf(self.file_object), "read and seek")
assert words == "should, should, uld, ding, fun, [ eof ]"
n = streambuf.default_buffer_size
soughts = self.file_object.seek_call_log
# stringent tests carefully crafted to make sure the seek-in-buffer
# optimisation works as expected
# C.f. the comment in the C++ function actual_write_test
assert soughts[-1] == (-4,2)
soughts = soughts[:-1]
if n >= 14:
assert soughts == []
else:
assert soughts[0] == (6,0)
soughts = soughts[1:]
if 8 <= n <= 13:
assert len(soughts) == 1 and self.only_seek_cur(soughts)
elif n == 7:
assert len(soughts) == 2 and self.only_seek_cur(soughts)
else:
assert soughts[0] == (6,0)
soughts = soughts[1:]
assert self.only_seek_cur(soughts)
if n == 4: assert len(soughts) == 1
else: assert len(soughts) == 2
self.file_object.close()
def to_smv_format_py(self,fileout,intfile_scale=0.0,debug_x=-1,debug_y=-1,
rotmat=False,extra=None,verbose=False,gz=False):
byte_order = "little_endian";
#recast the image file write to Python to afford extra options: rotmat, extra, gz
if gz:
from libtbx.smart_open import for_writing
outfile = for_writing(file_name=fileout+".gz", gzip_mode="wb")
else:
outfile = open(fileout,"wb");
outfile.write(("{\nHEADER_BYTES=1024;\nDIM=2;\nBYTE_ORDER=%s;\nTYPE=unsigned_short;\n"%byte_order).encode());
outfile.write(b"SIZE1=%d;\nSIZE2=%d;\nPIXEL_SIZE=%g;\nDISTANCE=%g;\n"%(
self.detpixels_fastslow[0],self.detpixels_fastslow[1],self.pixel_size_mm,self.distance_mm));
outfile.write(b"WAVELENGTH=%g;\n"%self.wavelength_A);
outfile.write(b"BEAM_CENTER_X=%g;\nBEAM_CENTER_Y=%g;\n"%self.beam_center_mm);
outfile.write(b"ADXV_CENTER_X=%g;\nADXV_CENTER_Y=%g;\n"%self.adxv_beam_center_mm);
outfile.write(b"MOSFLM_CENTER_X=%g;\nMOSFLM_CENTER_Y=%g;\n"%self.mosflm_beam_center_mm);
outfile.write(b"DENZO_X_BEAM=%g;\nDENZO_Y_BEAM=%g;\n"%self.denzo_beam_center_mm);
outfile.write(b"DIALS_ORIGIN=%g,%g,%g\n"%self.dials_origin_mm);
outfile.write(b"XDS_ORGX=%g;\nXDS_ORGY=%g;\n"%self.XDS_ORGXY);
outfile.write(b"CLOSE_DISTANCE=%g;\n"%self.close_distance_mm);
outfile.write(b"PHI=%g;\nOSC_START=%g;\nOSC_RANGE=%g;\n"%(self.phi_deg,self.phi_deg,self.osc_deg));
outfile.write(b"TIME=%g;\n"%self.exposure_s);
outfile.write(b"TWOTHETA=%g;\n"%self.detector_twotheta_deg);
outfile.write(b"DETECTOR_SN=000;\n");
outfile.write(b"ADC_OFFSET=%g;\n"%self.adc_offset_adu);
outfile.write(b"BEAMLINE=fake;\n");
if rotmat:
from scitbx.matrix import sqr
RSABC = sqr(self.Amatrix).inverse().transpose()
outfile.write( ("DIRECT_SPACE_ABC=%s;\n"%(",".join([repr(a) for a in RSABC.elems]))).encode() )
if extra is not None:
outfile.write(extra.encode())
outfile.write(b"}\f");
assert outfile.tell() < 1024, "SMV header too long, please edit this code and ask for more bytes."
while ( outfile.tell() < 1024 ): outfile.write(b" ")
from six import PY3
if PY3:
# Python3-compatible method for populating the output buffer.
# Py2 implementation is more elegant in that the streambuf may be passed to C++,
# and the data are gzipped in chunks (default 1024). Py3 will not accept this method
# as it is PyString-based, with no converter mechanisms to bring data into PyBytes.
# The Py3 method brings the full data in one chunk into PyBytes and then populates
# the output buffer in Python rather than C++.
image_bytes = self.raw_pixels_unsigned_short_as_python_bytes(intfile_scale,debug_x,debug_y)
ptr = 0; nbytes = len(image_bytes)
while (ptr < nbytes): # chunked output necessary to prevent intermittent MemoryError
outfile.write(image_bytes[ptr : min(ptr + 65536, nbytes)])
ptr += 65536
outfile.close();
return
from boost_adaptbx.boost.python import streambuf
self.to_smv_format_streambuf(streambuf(outfile),intfile_scale,debug_x,debug_y)
outfile.close();
def channel_pixels(wavelength_A, flux, N, UMAT_nm, Amatrix_rot, rank,
sfall_channel):
if rank in [0, 7]:
print("USING scatterer-specific energy-dependent scattering factors")
SIM = nanoBragg(detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
wavelength_A=wavelength_A,
verbose=0)
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
SIM.distance_mm = 141.7
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = wavelength_A
SIM.polarization = 1
SIM.default_F = 0
SIM.Fhkl = sfall_channel
SIM.Amatrix_RUB = Amatrix_rot
SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian
SIM.progress_meter = False
# flux is always in photons/s
SIM.flux = flux
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
if rank in [0, 7]: print("Ncells_abc=", SIM.Ncells_abc)
SIM.Ncells_abc = temp
if rank in [0, 7]: P = Profiler("nanoBragg C++ rank %d" % (rank))
if add_spots_algorithm == "NKS":
from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect
SIM.add_nanoBragg_spots_nks(streambuf(StringIO()))
elif add_spots_algorithm == "JH":
SIM.add_nanoBragg_spots()
elif add_spots_algorithm == "cuda":
SIM.xtal_shape = shapetype.Gauss_argchk
devices_per_node = int(os.environ["DEVICES_PER_NODE"])
SIM.device_Id = rank % devices_per_node
#if rank==7:
# os.system("nvidia-smi")
SIM.add_nanoBragg_spots_cuda()
else:
raise Exception("unknown spots algorithm")
if rank in [0, 7]: del P
return SIM
def __call__(self, N, UMAT_nm, Amatrix_rot, sfall_channel):
SIM = nanoBragg(detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
wavelength_A=self.wavlen[0],
verbose=0)
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.mosaic_domains = n_mosaic_domains # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
SIM.mosaic_spread_deg = mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.distance_mm = distance_mm
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.polarization = 1
SIM.default_F = 0
SIM.Fhkl = sfall_channel
SIM.Amatrix_RUB = Amatrix_rot
SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian
SIM.progress_meter = False
# flux is always in photons/s
#SIM.flux=0.8E10 # number of photons per 100 ps integrated pulse, 24-bunch APS
SIM.flux = 0.8E12 # number of photons per 100 ps integrated pulse, 24-bunch APS
SIM.xray_source_wavelengths_A = self.wavlen
SIM.xray_source_intensity_fraction = self.norm_intensity
SIM.xray_source_XYZ = self.source_XYZ
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
print("Ncells_abc=", SIM.Ncells_abc)
SIM.Ncells_abc = temp
from libtbx.development.timers import Profiler
P = Profiler("nanoBragg")
if add_spots_algorithm == "NKS":
print("USING NKS")
from boost_adaptbx.boost.python import streambuf # will deposit printout into dummy StringIO as side effect
SIM.add_nanoBragg_spots_nks(streambuf(StringIO()))
elif add_spots_algorithm == "JH":
print("USING JH")
SIM.add_nanoBragg_spots()
elif add_spots_algorithm == "cuda":
print("USING cuda")
SIM.add_nanoBragg_spots_cuda()
else:
raise Exception("unknown spots algorithm")
del P
return SIM
def exercise_read_failure(self):
self.create_file_object(mode='r')
self.file_object.close()
try:
ext.test_read(streambuf(self.file_object), "read")
except ValueError:
pass
else:
raise Exception_expected
self.file_object.close()
def exercise_write_failure(self):
import platform
if (platform.platform().find("redhat-8.0") >= 0):
return # avoid Abort
self.create_file_object(mode='r')
try:
ext.test_write(streambuf(self.file_object), "write")
except IOError as err:
pass
else:
raise Exception_expected
self.file_object.close()
def get_raw_data(self):
"""Read the data - assuming it is streaming 4-byte unsigned ints following the
header..."""
assert len(self.get_detector()) == 1
size = self.get_detector()[0].get_image_size()
f = self.open_file(self._image_file)
f.read(int(self._header_dictionary["HEADER_BYTES"]))
raw_data = read_int32(streambuf(f), int(size[0] * size[1]))
raw_data.reshape(flex.grid(size[1], size[0]))
return raw_data
def _read_raw_data(self, f):
"""Read raw data from a positioned file"""
# is this image a type we can read?
assert self._data_type in ["h", "f", "B", "l", "b", "H", "I", "d"]
if self._data_type == "f":
raw_data = read_float32(streambuf(f), self._image_num_elements)
elif self._data_type == "B":
raw_data = read_uint8(streambuf(f), self._image_num_elements)
elif self._data_type == "h":
raw_data = read_int16(streambuf(f), self._image_num_elements)
elif self._data_type == "H":
raw_data = read_uint16(streambuf(f), self._image_num_elements)
elif self._data_type == "l":
raw_data = read_int32(streambuf(f), self._image_num_elements)
elif self._data_type == "I":
raw_data = read_uint32(streambuf(f), self._image_num_elements)
# no C++ reader for remaining types (should be unusual anyway)
else:
vals = struct.unpack(
self._byteord + self._data_type * self._image_num_elements,
f.read(self._data_size * self._image_num_elements),
)
if self._data_type == "d":
raw_data = flex.double(vals)
if self._data_type in ["b", "?"]:
raw_data = flex.int(vals)
raw_data.reshape(flex.grid(self._image_size[1], self._image_size[0]))
return raw_data
def get_raw_data(self):
"""Get the pixel intensities (i.e. read the image and return as a
flex array of integers.)"""
# currently have no non-little-endian machines...
assert len(self.get_detector()) == 1
image_size = self.get_detector()[0].get_image_size()
with self.open_file(self._image_file) as fh:
fh.seek(self._header_size)
raw_data = read_uint16(streambuf(fh), int(image_size[0] * image_size[1]))
raw_data.reshape(flex.grid(image_size[1], image_size[0]))
return raw_data
def perform_one_simulation_optimized(self,model,ROI,models4):
# models4 is a dict of direct_matrices with keys "Amat","Amat_dx", "Amat_dy", "Amat_dz"
Amatrix_rot = models4[model]
self.SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
Amatrecover = sqr(self.SIM.Amatrix).transpose() # recovered Amatrix from SIM
Ori = crystal_orientation.crystal_orientation(Amatrecover, crystal_orientation.basis_type.reciprocal)
self.SIM.raw_pixels.fill(0.0) # effectively initializes the data pixels for a new simulation
temp_fill_zeroes_for_roi = self.SIM.raw_pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]]
channel_summation_roi_only = self.SIM.raw_pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]]
self.SIM.seed = 1
# simulated crystal is only 125 unit cells (25 nm wide)
# amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume)
output = StringIO() # open("myfile","w")
prefix = "key_slow_nonoise"
from LS49.work2_for_aca_lsq.util_partiality import response_plot
make_response_plot = response_plot(enable=False,title=prefix)
self.SIM.region_of_interest = ROI
from boost_adaptbx.boost.python import streambuf
self.SIM.printout=True # only for the purpose of getting the P1 Miller index and structure factor
fast = ROI[1][0] + (ROI[1][1]-ROI[1][0])//2
slow = ROI[0][0] + (ROI[0][1]-ROI[0][0])//2
self.SIM.printout_pixel_fastslow=(slow,fast)
for x in range(0,100,2): #len(flux)):
if self.flux[x]==0.0:continue
# initialize ROI-only to zero
self.SIM.raw_pixels.matrix_paste_block_in_place(temp_fill_zeroes_for_roi,ROI[1][0],ROI[0][0])
self.SIM.wavelength_A = self.wavlen[x]
self.SIM.flux = self.flux[x]
self.SIM.add_nanoBragg_spots_nks(streambuf(output))
self.SIM.printout = False # only do the printout once through
incremental_signal = self.SIM.raw_pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]] * self.crystal.domains_per_crystal
make_response_plot.append_channel_full_region(x,incremental_signal)
if x in [26,40,54,68]: # subsample 7096, 7110, 7124, 7138 eV
#print ("----------------------> subsample", x)
make_response_plot.incr_subsample_full_region(x,incremental_signal)
make_response_plot.increment_full_region(x,incremental_signal)
channel_summation_roi_only += incremental_signal;
message = output.getvalue().split()
miller = (int(message[4]),int(message[5]),int(message[6])) # nanoBragg P1 index
intensity = float(message[9]);
make_response_plot.plot(channel_summation_roi_only,miller)
return dict(roi_pixels=channel_summation_roi_only,miller=miller,intensity=intensity,
channels=make_response_plot.channels)
def _get_raw_data(self, index):
data_offset = self._header_dictionary["DataOffsetArray"][index]
with FormatSER.open_file(self._image_file, "rb") as f:
f.seek(data_offset)
d = {}
d["CalibrationOffsetX"] = struct.unpack("<d", f.read(8))[0]
d["CalibrationDeltaX"] = struct.unpack("<d", f.read(8))[0]
d["CalibrationElementX"] = struct.unpack("<I", f.read(4))[0]
d["CalibrationOffsetY"] = struct.unpack("<d", f.read(8))[0]
d["CalibrationDeltaY"] = struct.unpack("<d", f.read(8))[0]
d["CalibrationElementY"] = struct.unpack("<I", f.read(4))[0]
d["DataType"] = struct.unpack("<H", f.read(2))[0]
if d["DataType"] == 6:
read_pixel = dxtbx.ext.read_int32
elif d["DataType"] == 5:
read_pixel = dxtbx.ext.read_int16
elif d["DataType"] == 3:
read_pixel = dxtbx.ext.read_uint32
elif d["DataType"] == 2:
read_pixel = dxtbx.ext.read_uint16
elif d["DataType"] == 1:
read_pixel = dxtbx.ext.read_uint8
else:
raise RuntimeError(
"Image {} data is of an unsupported type".format(index + 1)
)
d["ArraySizeX"] = struct.unpack("<I", f.read(4))[0]
d["ArraySizeY"] = struct.unpack("<I", f.read(4))[0]
nelts = d["ArraySizeX"] * d["ArraySizeY"]
raw_data = read_pixel(streambuf(f), nelts)
# Check image size is as expected (same as the first image)
if d["ArraySizeX"] != self._header_dictionary["ArraySizeX"]:
raise RuntimeError(
"Image {} has an unexpected array size in X".format(index + 1)
)
if d["ArraySizeY"] != self._header_dictionary["ArraySizeY"]:
raise RuntimeError(
"Image {} has an unexpected array size in Y".format(index + 1)
)
image_size = (d["ArraySizeX"], d["ArraySizeY"])
raw_data.reshape(flex.grid(image_size[1], image_size[0]))
return raw_data
def get_raw_data(self):
"""Get the pixel intensities (i.e. read the image and return as a
flex array of integers.)"""
# currently have no non-little-endian machines...
assert self._tiff_byte_order == FormatTIFF.LITTLE_ENDIAN
bias = int(round(self._get_rayonix_bias()))
assert len(self.get_detector()) == 1
size = self.get_detector()[0].get_image_size()
f = FormatTIFF.open_file(self._image_file)
f.read(self._header_size)
raw_data = read_uint16(streambuf(f), int(size[0] * size[1])) - bias
image_size = self.get_detector()[0].get_image_size()
raw_data.reshape(flex.grid(image_size[1], image_size[0]))
return raw_data
def add_channel_pixels(self, wavelength_A, flux, rank, algo="cuda"):
"""
:param wavelength_A:
:param flux:
:param rank:
:param algo:
:return:
"""
self.SIM.flux = flux
self.SIM.wavelength_A = wavelength_A
P = Profiler("nanoBragg C++ rank %d" % (rank))
if algo == "NKS":
self.SIM.add_nanoBragg_spots_nks(streambuf(StringIO()))
self.raw_pixels = self.SIM.raw_pixels # NOTE: this actually incremenents hack for now, because cuda doesnt add spots
elif algo == "JH":
self.SIM.add_nanoBragg_spots()
self.raw_pixels = self.SIM.raw_pixels
elif algo == "cuda":
self.SIM.add_nanoBragg_spots_cuda()
#self.SIM.add_nanoBragg_spots_cuda_light() # FIXME: this should be updated to not initialize
self.raw_pixels += self.SIM.raw_pixels # NOTE: will be on GPU
else:
raise Exception("unknown spots algorithm '%s' " % algo)
def get_raw_data(self, index):
# is this image a type we can read?
assert self._data_type in ["h", "f", "B", "l", "b", "H", "I", "d"]
with FormatGatanDM4.open_file(self._image_file, "rb") as f:
f.seek(self._data_offset)
skip_bytes = index * self._image_num_elements * self._data_size
f.seek(skip_bytes, whence=1)
if self._data_type == "f":
raw_data = read_float32(streambuf(f), self._image_num_elements)
elif self._data_type == "B":
raw_data = read_uint8(streambuf(f), self._image_num_elements)
elif self._data_type == "h":
raw_data = read_int16(streambuf(f), self._image_num_elements)
elif self._data_type == "H":
raw_data = read_uint16(streambuf(f), self._image_num_elements)
elif self._data_type == "l":
raw_data = read_int32(streambuf(f), self._image_num_elements)
elif self._data_type == "I":
raw_data = read_uint32(streambuf(f), self._image_num_elements)
# no C++ reader for remaining types (should be unusual anyway)
else:
vals = struct.unpack(
self._byteord + self._data_type * self._image_num_elements,
f.read(self._data_size * self._image_num_elements),
)
if self._data_type == "d":
raw_data = flex.double(vals)
if self._data_type in ["b", "?"]:
raw_data = flex.int(vals)
raw_data.reshape(flex.grid(self._image_size[1], self._image_size[0]))
return raw_data
def read_from_stdin():
written = ext.test_read(streambuf(sys.stdin), "read")
assert written == "Veni, Vidi, Vici, [ fail, eof ]", written
def exercise_read(self): self.create_file_object(mode='r') words = ext.test_read(streambuf(self.file_object), "read") assert words == "Coding, should, be, fun, [ fail, eof ]" self.file_object.close()
def get_raw_data(self):
"""Get the pixel intensities (i.e. read the image and return as a
flex array of integers.)"""
# It is better to catch FORMAT 86 here and fail with a sensible error msg
# as soon as something is attempted with the image data rather than in
# the understand method. Otherwise the user gets FormatBruker reading the
# image improperly but without failing
if self.header_dict["FORMAT"] != "100":
raise NotImplementedError(
"Only FORMAT 100 images from the Photon II are currently supported"
)
f = self.open_file(self._image_file, "rb")
header_size = int(self.header_dict["HDRBLKS"]) * 512
f.read(header_size)
if is_big_endian():
read_2b = read_uint16_bs
read_4b = read_uint32_bs
else:
read_2b = read_uint16
read_4b = read_uint32
# NPIXELB stores the number of bytes/pixel for the data and the underflow
# table. We expect 1 byte for underflows and either 2 or 1 byte per pixel
# for the data
npixelb = [int(e) for e in self.header_dict["NPIXELB"].split()]
assert npixelb[1] == 1
if npixelb[0] == 1:
read_data = read_uint8
elif npixelb[0] == 2:
read_data = read_2b
else:
raise IncorrectFormatError(
"{} bytes per pixel is not supported".format(npixelb[0]))
nrows = int(self.header_dict["NROWS"].split()[0])
ncols = int(self.header_dict["NCOLS"].split()[0])
raw_data = read_data(streambuf(f), nrows * ncols)
image_size = (nrows, ncols)
raw_data.reshape(flex.grid(*image_size))
(num_underflows, num_2b_overflows, num_4b_overflows) = [
int(e) for e in self.header_dict["NOVERFL"].split()
]
# read underflows
if num_underflows > 0:
# stored values are padded to a multiple of 16 bytes
nbytes = num_underflows + 15 & ~(15)
underflow_vals = read_uint8(streambuf(f), nbytes)[:num_underflows]
else:
underflow_vals = None
# handle 2 byte overflows
if num_2b_overflows > 0:
# stored values are padded to a multiple of 16 bytes
nbytes = num_2b_overflows * 2 + 15 & ~(15)
overflow_vals = read_2b(streambuf(f),
nbytes // 2)[:num_2b_overflows]
overflow = flex.int(nrows * ncols, 0)
sel = (raw_data == 255).as_1d()
overflow.set_selected(sel, overflow_vals - 255)
overflow.reshape(flex.grid(*image_size))
raw_data += overflow
# handle 4 byte overflows
if num_4b_overflows > 0:
# stored values are padded to a multiple of 16 bytes
nbytes = num_4b_overflows * 4 + 15 & ~(15)
overflow_vals = read_4b(streambuf(f),
nbytes // 4)[:num_4b_overflows]
overflow = flex.int(nrows * ncols, 0)
sel = (raw_data == 65535).as_1d()
overflow.set_selected(sel, overflow_vals - 65535)
overflow.reshape(flex.grid(*image_size))
raw_data += overflow
# handle underflows
if underflow_vals is not None:
sel = (raw_data == 0).as_1d()
underflow = flex.int(nrows * ncols, 0)
underflow.set_selected(sel, underflow_vals)
underflow.reshape(flex.grid(*image_size))
raw_data += underflow
# handle baseline. num_underflows == -1 means no baseline subtraction. See
# https://github.com/cctbx/cctbx_project/files/1262952/BISFrameFileFormats.zip
if num_underflows != -1:
num_exposures = [int(e) for e in self.header_dict["NEXP"].split()]
baseline = num_exposures[2]
raw_data += baseline
return raw_data
