def write_image(out_image, pixel_values, header, nn=1):
from cbflib_adaptbx import compress
assert not os.path.exists(out_image)
start_tag = binascii.unhexlify("0c1a04d5")
compressed = compress(pixel_values)
fixed_header = ""
header = header.decode()
for record in header.split("\n")[:-1]:
if "X-Binary-Size:" in record:
fixed_header += f"X-Binary-Size: {len(compressed)}\r\n"
elif "Content-MD5" in record:
pass
elif "Count_cutoff" in record:
cutoff = int(record.split()[2]) * nn
fixed_header += "# Count_cutoff %d counts\n" % cutoff
else:
fixed_header += f"{record}\n"
tailer = "\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n"
gz_open(out_image, "wb").write(fixed_header.encode() + start_tag +
compressed + tailer.encode())
def write_image_from_flex_array(out_image, pixel_values, header):
'''Write a scaled CBF image from an array of pixel values and a header to
add at the top. N.B. clobbers the binary size of the compressed data &
the MD5 hash of the data.'''
from cbflib_adaptbx import compress
import binascii
import os
assert(not os.path.exists(out_image))
start_tag = binascii.unhexlify('0c1a04d5')
compressed = compress(pixel_values)
fixed_header = ''
for record in header.split('\n')[:-1]:
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed)
elif 'Content-MD5' in record:
pass
else:
fixed_header += '%s\n' % record
tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n'
open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def write_image_from_flex_array(out_image, pixel_values, header):
'''Write a scaled CBF image from an array of pixel values and a header to
add at the top. N.B. clobbers the binary size of the compressed data &
the MD5 hash of the data.'''
from cbflib_adaptbx import compress
import binascii
import os
assert (not os.path.exists(out_image))
start_tag = binascii.unhexlify('0c1a04d5')
compressed = compress(pixel_values)
fixed_header = ''
for record in header.split('\n')[:-1]:
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed)
elif 'Content-MD5' in record:
pass
else:
fixed_header += '%s\n' % record
tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n'
open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def write_image_from_flex_array(out_image, pixel_values, header):
"""Write a scaled CBF image from an array of pixel values and a header to
add at the top. N.B. clobbers the binary size of the compressed data &
the MD5 hash of the data."""
from cbflib_adaptbx import compress
assert not os.path.exists(out_image)
start_tag = binascii.unhexlify("0c1a04d5")
compressed = compress(pixel_values)
fixed_header = ""
header = header.decode()
for record in header.split("\n")[:-1]:
if "X-Binary-Size:" in record:
fixed_header += f"X-Binary-Size: {len(compressed)}\r\n"
elif "Content-MD5" in record:
pass
else:
fixed_header += f"{record}\n"
tailer = "\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n"
open(out_image, "wb").write(
fixed_header.encode() + start_tag + compressed + tailer.encode()
)
def recompute_BKGINIT(bkginit_in, init_lp, bkginit_out):
start_tag = binascii.unhexlify('0c1a04d5')
data = open(bkginit_in, 'rb').read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
for record in cbf_header.split('\n'):
if 'X-Binary-Size-Fastest-Dimension' in record:
fast = int(record.split()[-1])
elif 'X-Binary-Size-Second-Dimension' in record:
slow = int(record.split()[-1])
elif 'X-Binary-Number-of-Elements' in record:
length = int(record.split()[-1])
assert(length == fast * slow)
pixel_values = uncompress(packed = data[data_offset:],
fast = fast, slow = slow)
untrusted = []
for record in open(init_lp):
if 'UNTRUSTED_RECTANGLE=' in record:
untrusted.append(map(int, record.replace('.', ' ').split()[1:5]))
modified_pixel_values = copy.deepcopy(pixel_values)
for s in range(5, slow - 5):
y = s + 1
for f in range(5, fast - 5):
x = f + 1
trusted = True
for x0, x1, y0, y1 in untrusted:
if (x >= x0) and (x <= x1) and (y >= y0) and (y <= y1):
trusted = False
break
if trusted:
pixel = pixel_values[s * fast + f]
if pixel < 0:
pixels = []
for j in range(-2, 3):
for i in range(-2, 3):
p = pixel_values[(s + j) * fast + f + i]
if p > 0:
pixels.append(p)
modified_pixel_values[s * fast + f] = int(
sum(pixels) / len(pixels))
open(bkginit_out, 'wb').write(cbf_header + start_tag +
compress(modified_pixel_values))
return
def recompute_BKGINIT(bkginit_in, init_lp, bkginit_out):
start_tag = binascii.unhexlify('0c1a04d5')
data = open(bkginit_in, 'rb').read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
for record in cbf_header.split('\n'):
if 'X-Binary-Size-Fastest-Dimension' in record:
fast = int(record.split()[-1])
elif 'X-Binary-Size-Second-Dimension' in record:
slow = int(record.split()[-1])
elif 'X-Binary-Number-of-Elements' in record:
length = int(record.split()[-1])
assert (length == fast * slow)
pixel_values = uncompress(packed=data[data_offset:], fast=fast, slow=slow)
untrusted = []
for record in open(init_lp):
if 'UNTRUSTED_RECTANGLE=' in record:
untrusted.append(map(int, record.replace('.', ' ').split()[1:5]))
modified_pixel_values = copy.deepcopy(pixel_values)
for s in range(5, slow - 5):
y = s + 1
for f in range(5, fast - 5):
x = f + 1
trusted = True
for x0, x1, y0, y1 in untrusted:
if (x >= x0) and (x <= x1) and (y >= y0) and (y <= y1):
trusted = False
break
if trusted:
pixel = pixel_values[s * fast + f]
if pixel < 0:
pixels = []
for j in range(-2, 3):
for i in range(-2, 3):
p = pixel_values[(s + j) * fast + f + i]
if p > 0:
pixels.append(p)
modified_pixel_values[s * fast + f] = int(
sum(pixels) / len(pixels))
open(bkginit_out,
'wb').write(cbf_header + start_tag + compress(modified_pixel_values))
return
def squishGain(cbf_file, out_name, force_gain=None):
start_tag = binascii.unhexlify("0c1a04d5")
data = cbf_file.read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
for record in cbf_header.split("\n"):
if "X-Binary-Size-Fastest-Dimension" in record:
fast = int(record.split()[-1])
elif "X-Binary-Size-Second-Dimension" in record:
slow = int(record.split()[-1])
elif "X-Binary-Size:" in record:
xbsize_record = record
length = int(record.split()[-1])
values = uncompress(packed=data[data_offset:data_offset + length],
fast=fast,
slow=slow)
# remainder of the file, contains another CIF-BINARY-FORMAT-SECTION that looks
# like just zero padding.
tail = data[data_offset + length:]
modified = copy.deepcopy(values)
if force_gain:
new_val = int(1000 * force_gain)
else:
# calculate the mean of values that are greater than zero. This is close
# to 1000 times the "MEAN GAIN VALUE" reported in INIT.LP
dval1d = modified.as_1d().as_double()
mval = flex.mean(dval1d.select(dval1d > 0))
new_val = int(mval)
# Set this value everywhere that is not a masked value marked by -3
print("Setting flat gain of {0}".format(new_val / 1000.0))
modified.set_selected(modified >= 0, new_val)
# Compress the data
compressed = compress(modified)
nbytes = len(compressed)
# Update the header
pre, post = cbf_header.split(xbsize_record)
new_xbsize_record = "X-Binary-Size:{0:10d}".format(nbytes)
if xbsize_record.endswith("\r"):
new_xbsize_record += "\r"
new_cbf_header = pre + new_xbsize_record + post
# Write the file out.
open(out_name, "wb").write(new_cbf_header + start_tag + compressed + tail)
def save_cbf(im, orig_path, out_cbf=None, pOpt=False):
''' convert np.ndarray to cbf
- im : np.ndarray or str : image to convert to cbf
- orig_path : the path to exp data to get the header from those images
- out_cbf : name of the .cbf output file
- pOpt : Show cbf file with adxv if True (remember to set the path to adxv l.13)
'''
from utils import glob_colors as colors
if isinstance(im, str):
im = np.load(im)
if not out_cbf: out_cbf = in_npy.replace('.npy', '.cbf')
orig_file = glob(orig_path + '*.cbf')[1] #;print(orig_file)
# orig_file = "/home/tarik/Documents/data/ireloh/IRELOH_ED_Dataset_1/n14_a004_0484.cbf"
start_tag = binascii.unhexlify("0c1a04d5")
with open(orig_file, 'rb') as cbf:
data = cbf.read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
for record in cbf_header.decode().split("\n"):
if "X-Binary-Size-Fastest-Dimension" in record:
fast = int(record.split()[-1])
elif "X-Binary-Size-Second-Dimension" in record:
slow = int(record.split()[-1])
elif "X-Binary-Size:" in record:
xbsize_record = record
length = int(record.split()[-1])
tail = data[data_offset + length:]
# print(im.shape,fast,slow,length)
im001 = flex.int32(im) #np.array(im,dtype=np.int32))
compressed = compress(im001)
nbytes = len(compressed)
# Update the header
pre, post = cbf_header.decode().split(xbsize_record)
new_xbsize_record = "X-Binary-Size:{0:10d}".format(nbytes)
if xbsize_record.endswith("\r"):
new_xbsize_record += "\r"
new_cbf_header = pre + new_xbsize_record + post
open(out_cbf,
"wb").write(new_cbf_header.encode() + start_tag + compressed + tail)
print(colors.green + 'file saved : \n' + colors.yellow + out_cbf +
colors.black)
if pOpt:
p = Popen("%s %s" % (adxv, out_cbf), shell=True)
p.wait()
def multiplyCounts(cbf_file, out_name, multiplier):
start_tag = binascii.unhexlify("0c1a04d5")
data = cbf_file.read()
data_offset = data.find(start_tag) + 4
cbf_header = data[:data_offset - 4]
fast = 0
slow = 0
length = 0
for record in cbf_header.split("\n"):
if "X-Binary-Size-Fastest-Dimension" in record:
fast = int(record.split()[-1])
elif "X-Binary-Size-Second-Dimension" in record:
slow = int(record.split()[-1])
elif "X-Binary-Size:" in record:
xbsize_record = record
length = int(record.split()[-1])
values = uncompress(packed=data[data_offset:data_offset + length],
fast=fast,
slow=slow)
# remainder of the file, contains another CIF-BINARY-FORMAT-SECTION that looks
# like just zero padding.
tail = data[data_offset + length:]
# multiply all positive values
modified = copy.deepcopy(values).as_1d()
sel = modified > 0
new_val = modified.select(sel) * multiplier
modified.set_selected(sel, new_val)
# reshape
modified.reshape(values.accessor())
# Compress the data
compressed = compress(modified)
nbytes = len(compressed)
# Update the header
pre, post = cbf_header.split(xbsize_record)
new_xbsize_record = "X-Binary-Size:{0:10d}".format(nbytes)
if xbsize_record.endswith("\r"):
new_xbsize_record += "\r"
new_cbf_header = pre + new_xbsize_record + post
# Write the file out.
open(out_name, "wb").write(new_cbf_header + start_tag + compressed + tail)
def add_zingers(imageset, params):
from dxtbx.format.FormatCBF import FormatCBF
assert issubclass(imageset.reader().get_format_class(),
FormatCBF), "Only CBF format images supported"
from cbflib_adaptbx import compress
import binascii
from numpy.random import poisson
from random import sample
import os.path
for i in range(len(imageset)):
image_data = imageset[i]
num = poisson(params.zingers.average_per_image)
index = sample(range(len(image_data)), num)
value = list(poisson(params.zingers.average_intensity, num))
for j, v in zip(index, value):
image_data[j] += v
out_image = os.path.join(params.output.directory, "image_%04i.cbf" % i)
start_tag = binascii.unhexlify("0c1a04d5")
data = open(imageset.get_path(i), "rb").read()
data_offset = data.find(start_tag)
cbf_header = data[:data_offset]
new_header = []
compressed = compress(image_data)
old_size = 0
for record in cbf_header.split("\n")[:-1]:
if "X-Binary-Size:" in record:
old_size = int(record.split()[-1])
new_header.append("X-Binary-Size: %d\r\n" % len(compressed))
elif "Content-MD5" in record:
pass
else:
new_header.append("%s\n" % record)
tailer = data[data_offset + 4 + old_size:]
with open(out_image, "wb") as f:
f.write("".join(new_header) + start_tag + compressed + tailer)
print("%s written with %d zingers" % (out_image, num))
return
def add_zingers(imageset, params):
from dxtbx.format.FormatCBF import FormatCBF
assert issubclass(imageset.reader().get_format_class(), FormatCBF), (
"Only CBF format images supported")
from cbflib_adaptbx import compress
import binascii
from numpy.random import poisson
from random import sample
import os.path
for i in range(len(imageset)):
image_data = imageset[i]
num = poisson(params.zingers.average_per_image)
index = sample(range(len(image_data)), num)
value = list(poisson(params.zingers.average_intensity, num))
for j, v in zip(index, value):
image_data[j] += v
out_image = os.path.join(params.output.directory, "image_%04i.cbf" % i)
start_tag = binascii.unhexlify('0c1a04d5')
data = open(imageset.get_path(i), 'rb').read()
data_offset = data.find(start_tag)
cbf_header = data[:data_offset]
new_header = []
compressed = compress(image_data)
old_size = 0
for record in cbf_header.split('\n')[:-1]:
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
new_header.append('X-Binary-Size: %d\r\n' % len(compressed))
elif 'Content-MD5' in record:
pass
else:
new_header.append('%s\n' % record)
tailer = data[data_offset + 4 + old_size:]
with open(out_image, 'wb') as f:
f.write(''.join(new_header) + start_tag + compressed + tailer)
print '%s written with %d zingers' % (out_image, num)
return
def basic_tests(verbose=True):
initial_intdata = create_random_data_with_gaussian_distribution(0.0,100.0)
#special deltas to test the compression algorithm
addresses = [3,6,9,12,15,18]
deltas = [-127,128,-32767,32768,-2147483647,2147483647]
for x in xrange(6):
initial_intdata[addresses[x]-1]=0
initial_intdata[addresses[x]]=deltas[x]
if verbose: P=Profiler("compress")
array_shape = initial_intdata.focus()
if verbose: print array_shape
compressed = compress(initial_intdata)
if verbose: print len(compressed)
if verbose: P=Profiler("uncompress")
decompressed_dat = uncompress(packed=compressed, fast=array_shape[1], slow=array_shape[0])
if verbose: del P
assert assert_equal(initial_intdata, decompressed_dat)
def write_image(out_image, pixel_values, header):
from cbflib_adaptbx import compress
assert not os.path.exists(out_image)
start_tag = binascii.unhexlify("0c1a04d5")
compressed = compress(pixel_values)
fixed_header = ""
for record in header.split("\n")[:-1]:
if "X-Binary-Size:" in record:
fixed_header += "X-Binary-Size: %d\r\n" % len(compressed)
elif "Content-MD5" in record:
pass
else:
fixed_header += "%s\n" % record
tailer = "\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n"
gz_open(out_image,
"wb").write(fixed_header + start_tag + compressed + tailer)
def basic_tests(verbose=True):
initial_intdata = create_random_data_with_gaussian_distribution(0.0, 100.0)
#special deltas to test the compression algorithm
addresses = [3, 6, 9, 12, 15, 18]
deltas = [-127, 128, -32767, 32768, -2147483647, 2147483647]
for x in range(6):
initial_intdata[addresses[x] - 1] = 0
initial_intdata[addresses[x]] = deltas[x]
if verbose: P = Profiler("compress")
array_shape = initial_intdata.focus()
if verbose: print(array_shape)
compressed = compress(initial_intdata)
if verbose: print(len(compressed))
if verbose: P = Profiler("uncompress")
decompressed_dat = uncompress(packed=compressed,
fast=array_shape[1],
slow=array_shape[0])
if verbose: del P
assert assert_equal(initial_intdata, decompressed_dat)
def write_image(out_image, pixel_values, header):
from cbflib_adaptbx import compress
import binascii
import os
assert(not os.path.exists(out_image))
start_tag = binascii.unhexlify('0c1a04d5')
compressed = compress(pixel_values)
fixed_header = ''
for record in header.split('\n')[:-1]:
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed)
elif 'Content-MD5' in record:
pass
else:
fixed_header += '%s\n' % record
tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n'
gz_open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def write_image(out_image, pixel_values, header):
from cbflib_adaptbx import compress
import binascii
import os
assert(not os.path.exists(out_image))
start_tag = binascii.unhexlify('0c1a04d5')
compressed = compress(pixel_values)
fixed_header = ''
for record in header.split('\n')[:-1]:
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed)
elif 'Content-MD5' in record:
pass
else:
fixed_header += '%s\n' % record
tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n'
gz_open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def merge_cbf(imageset, n_images, out_prefix="sum_"):
from dxtbx.format.FormatCBF import FormatCBF
assert issubclass(imageset.get_format_class(), FormatCBF), (
"Only CBF format images supported")
from cbflib_adaptbx import compress
import binascii
assert len(imageset) >= n_images
n_output_images = len(imageset) // n_images
in_oscillation = imageset.get_scan().get_oscillation()[1]
out_oscillation = in_oscillation * n_images
for i_out in range(n_output_images):
data_out = None
for j in range(n_images):
i_in = (i_out*n_images) + j
data_in = imageset.get_raw_data(i_in)
assert len(data_in) == 1
data_in = data_in[0]
if data_out is None:
data_out = data_in
else:
# FIXME only add pixels to this which are > 0; image pixels < 0
# are meaningful and should be preserved;
# Achieved by setting -ve values here to 0 before +=
# This assumes that -ve values are constant over all images
data_special = data_in < 0
data_in.set_selected(data_special, 0)
data_out += data_in
out_image = "%s%04i.cbf" %(out_prefix, i_out+1)
start_tag = binascii.unhexlify('0c1a04d5')
data = open(imageset.get_path(i_out*n_images), 'rb').read()
data_offset = data.find(start_tag)
cbf_header = data[:data_offset]
new_header = []
compressed = compress(data_out)
old_size = 0
for record in cbf_header.split('\n')[:-1]:
rsplit = record.split(' ')
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
new_header.append('X-Binary-Size: %d\r\n' % len(compressed))
elif 'Content-MD5' in record:
pass
elif len(rsplit) > 3 and rsplit[1] in { \
'Exposure_time', 'Angle_increment', 'Exposure_period', 'Count_cutoff', \
'Phi_increment', 'Omega_increment', 'Chi_increment' }:
if rsplit[1] == 'Count_cutoff': # needs to be an integer
new_header.append('%s\n' % ' '.join(rsplit[:2] + ['%d' % (n_images * int(rsplit[2]))] + rsplit[3:]))
else:
new_header.append('%s\n' % ' '.join(rsplit[:2] + ['%f' % (n_images * float(rsplit[2]))] + rsplit[3:]))
else:
new_header.append('%s\n' % record)
loop_lines = [n for n, record in enumerate(new_header) if record.startswith('loop_')]
multiply_fields = { '_diffrn_scan_axis.angle_range', '_diffrn_scan_axis.angle_increment', \
'_diffrn_scan_axis.displacement_range', '_diffrn_scan_axis.displacement_increment', \
'_diffrn_scan_frame.integration_time', '_diffrn_scan_frame.exposure_time', \
'_array_intensities.overload' }
for loop_start in loop_lines:
n = loop_start
modifiers = []
while True:
n = n + 1
line = new_header[n].strip()
if line in { '', ';' }: # end of loop
break
elif line.startswith('_'): # loop header
if line in multiply_fields:
modifiers.append(n_images)
else:
modifiers.append(None)
elif any(modifiers): # loop body
# NOTE: This can break when fields are modified in loops with
# 'Strings with spaces, as they are seen as multiple columns, or with'
# _multiple _columns _defined _on _same _line _they _are _seen _as _one _column
new_line = [ element if modifier is None else '%f' % (float(element) * modifier) \
for modifier, element in zip(modifiers, line.split()) ]
new_header[n] = '%s\r\n' % ' '.join(new_line)
tailer = data[data_offset + 4 + old_size:]
with open(out_image, 'wb') as f:
f.write(''.join(new_header) + start_tag + compressed + tailer)
print '%s written' % out_image
return
def merge_cbf(imageset, n_images, out_prefix="sum_", get_raw_data_from_imageset=True):
from dxtbx.format.FormatCBF import FormatCBF
assert issubclass(
imageset.get_format_class(), FormatCBF
), "Only CBF format images supported"
from cbflib_adaptbx import compress
assert len(imageset) >= n_images
n_output_images = len(imageset) // n_images
n_digits = len(str(n_output_images))
for i_out in range(n_output_images):
data_out = None
for j in range(n_images):
i_in = (i_out * n_images) + j
if get_raw_data_from_imageset:
data_in = imageset.get_raw_data(i_in)
else:
data_in = get_raw_data_from_file(imageset, i_in)
assert len(data_in) == 1
data_in = data_in[0]
if data_out is None:
data_out = data_in
else:
# FIXME only add pixels to this which are > 0; image pixels < 0
# are meaningful and should be preserved;
# Achieved by setting -ve values here to 0 before +=
# This assumes that -ve values are constant over all images
data_special = data_in < 0
data_in.set_selected(data_special, 0)
data_out += data_in
out_image = "{prefix}{number:0{digits}d}.cbf".format(
prefix=out_prefix, number=i_out + 1, digits=n_digits
)
start_tag = binascii.unhexlify("0c1a04d5")
with open(imageset.get_path(i_out * n_images), "rb") as fh:
data = fh.read()
data_offset = data.find(start_tag)
cbf_header = data[:data_offset].decode("latin-1")
new_header = []
compressed = compress(data_out)
old_size = 0
for record in cbf_header.split("\n")[:-1]:
rsplit = record.split(" ")
if "X-Binary-Size:" in record:
old_size = int(record.split()[-1])
new_header.append(f"X-Binary-Size: {len(compressed)}\r\n")
elif "Content-MD5" in record:
pass
elif len(rsplit) > 3 and rsplit[1] in {
"Exposure_time",
"Angle_increment",
"Exposure_period",
"Count_cutoff",
"Phi_increment",
"Omega_increment",
"Chi_increment",
}:
if rsplit[1] == "Count_cutoff": # needs to be an integer
new_header.append(
"%s\n"
% " ".join(
rsplit[:2]
+ ["%d" % (n_images * int(rsplit[2]))]
+ rsplit[3:]
)
)
else:
new_header.append(
"%s\n"
% " ".join(
rsplit[:2]
+ ["%f" % (n_images * float(rsplit[2]))]
+ rsplit[3:]
)
)
else:
new_header.append(f"{record}\n")
loop_lines = [
n for n, record in enumerate(new_header) if record.startswith("loop_")
]
multiply_fields = {
"_diffrn_scan_axis.angle_range",
"_diffrn_scan_axis.angle_increment",
"_diffrn_scan_axis.displacement_range",
"_diffrn_scan_axis.displacement_increment",
"_diffrn_scan_frame.integration_time",
"_diffrn_scan_frame.exposure_time",
"_array_intensities.overload",
}
for loop_start in loop_lines:
n = loop_start
modifiers = []
while True:
n = n + 1
line = new_header[n].strip()
if line in {"", ";"}: # end of loop
break
elif line.startswith("_"): # loop header
if line in multiply_fields:
modifiers.append(n_images)
else:
modifiers.append(None)
elif any(modifiers): # loop body
# NOTE: This can break when fields are modified in loops with
# 'Strings with spaces, as they are seen as multiple columns, or with'
# _multiple _columns _defined _on _same _line _they _are _seen _as _one _column
new_line = [
element
if modifier is None
else f"{float(element) * modifier:f}"
for modifier, element in zip(modifiers, line.split())
]
new_header[n] = f"{' '.join(new_line)}\r\n"
tailer = data[data_offset + 4 + old_size :]
with open(out_image, "wb") as f:
f.write("".join(new_header).encode("latin-1"))
f.write(start_tag)
f.write(compressed)
f.write(tailer)
print(f"{out_image} written")
def make_cbf(in_name, template):
f = h5py.File(in_name, "r")
depends_on(f)
mask = None
start_tag = binascii.unhexlify("0c1a04d5")
for j in range(len(f["/entry/sample/transformations/omega"][()])):
block = 1 + (j // 1000)
i = j % 1000
header = compute_cbf_header(f, j)
depth, height, width = f["/entry/data/data_%06d" % block].shape
data = flex.int(numpy.int32(f["/entry/data/data_%06d" % block][i]))
good = data.as_1d() < 65535
data.as_1d().set_selected(~good, -2)
# set the tile join regions to -1 - MOSFLM cares about this apparently
if mask is None:
mask = get_mask(width, height)
data.as_1d().set_selected(mask.as_1d(), -1)
compressed = compress(data)
mime = """
_array_data.data
;
--CIF-BINARY-FORMAT-SECTION--
Content-Type: application/octet-stream;
conversions="x-CBF_BYTE_OFFSET"
Content-Transfer-Encoding: BINARY
X-Binary-Size: %d
X-Binary-ID: 1
X-Binary-Element-Type: "signed 32-bit integer"
X-Binary-Element-Byte-Order: LITTLE_ENDIAN
X-Binary-Number-of-Elements: %d
X-Binary-Size-Fastest-Dimension: %d
X-Binary-Size-Second-Dimension: %d
X-Binary-Size-Padding: 4095
""" % (
len(compressed),
data.size(),
data.focus()[1],
data.focus()[0],
)
padding = (bytearray(4095) + b"""--CIF-BINARY-FORMAT-SECTION----
;""")
with open(template % (j + 1), "wb") as fout:
print(template % (j + 1))
fout.write(
("".join(header) + mime).replace("\n",
"\r\n").encode("latin-1"))
fout.write(start_tag)
fout.write(compressed)
fout.write(padding)
f.close()
def merge_cbf(imageset, n_images, out_prefix="sum_"):
from dxtbx.format.FormatCBF import FormatCBF
assert issubclass(imageset.reader().get_format_class(), FormatCBF), (
"Only CBF format images supported")
from cbflib_adaptbx import compress
import binascii
assert len(imageset) >= n_images
n_output_images = len(imageset) // n_images
in_oscillation = imageset.get_scan().get_oscillation()[1]
out_oscillation = in_oscillation * n_images
for i_out in range(n_output_images):
data_out = None
for j in range(n_images):
i_in = (i_out*n_images) + j
data_in = imageset.get_raw_data(i_in)
assert len(data_in) == 1
data_in = data_in[0]
if data_out is None:
data_out = data_in
else:
# FIXME only add pixels to this which are > 0; image pixels < 0
# are meaningful and should be preserved;
# Achieved by setting -ve values here to 0 before +=
# This assumes that -ve values are constant over all images
data_special = data_in < 0
data_in.set_selected(data_special, 0)
data_out += data_in
out_image = "%s%04i.cbf" %(out_prefix, i_out+1)
start_tag = binascii.unhexlify('0c1a04d5')
data = open(imageset.get_path(i_out*n_images), 'rb').read()
data_offset = data.find(start_tag)
cbf_header = data[:data_offset]
new_header = []
compressed = compress(data_out)
old_size = 0
for record in cbf_header.split('\n')[:-1]:
rsplit = record.split(' ')
if 'X-Binary-Size:' in record:
old_size = int(record.split()[-1])
new_header.append('X-Binary-Size: %d\r\n' % len(compressed))
elif 'Content-MD5' in record:
pass
elif len(rsplit) > 3 and rsplit[1] in { \
'Exposure_time', 'Angle_increment', 'Exposure_period', 'Count_cutoff', \
'Phi_increment', 'Omega_increment', 'Chi_increment' }:
new_header.append('%s\n' % ' '.join(rsplit[:2] + ['%f' % (n_images * float(rsplit[2]))] + rsplit[3:]))
else:
new_header.append('%s\n' % record)
loop_lines = [n for n, record in enumerate(new_header) if record.startswith('loop_')]
multiply_fields = { '_diffrn_scan_axis.angle_range', '_diffrn_scan_axis.angle_increment', \
'_diffrn_scan_axis.displacement_range', '_diffrn_scan_axis.displacement_increment', \
'_diffrn_scan_frame.integration_time', '_diffrn_scan_frame.exposure_time', \
'_array_intensities.overload' }
for loop_start in loop_lines:
n = loop_start
modifiers = []
while True:
n = n + 1
line = new_header[n].strip()
if line in { '', ';' }: # end of loop
break
elif line.startswith('_'): # loop header
if line in multiply_fields:
modifiers.append(n_images)
else:
modifiers.append(None)
elif any(modifiers): # loop body
# NOTE: This can break when fields are modified in loops with
# 'Strings with spaces, as they are seen as multiple columns, or with'
# _multiple _columns _defined _on _same _line _they _are _seen _as _one _column
new_line = [ element if modifier is None else '%f' % (float(element) * modifier) \
for modifier, element in zip(modifiers, line.split()) ]
new_header[n] = '%s\r\n' % ' '.join(new_line)
tailer = data[data_offset + 4 + old_size:]
with open(out_image, 'wb') as f:
f.write(''.join(new_header) + start_tag + compressed + tailer)
print '%s written' % out_image
return