def write_image(out_image, pixel_values, header, nn=1): from cbflib_adaptbx import compress assert not os.path.exists(out_image) start_tag = binascii.unhexlify("0c1a04d5") compressed = compress(pixel_values) fixed_header = "" header = header.decode() for record in header.split("\n")[:-1]: if "X-Binary-Size:" in record: fixed_header += f"X-Binary-Size: {len(compressed)}\r\n" elif "Content-MD5" in record: pass elif "Count_cutoff" in record: cutoff = int(record.split()[2]) * nn fixed_header += "# Count_cutoff %d counts\n" % cutoff else: fixed_header += f"{record}\n" tailer = "\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n" gz_open(out_image, "wb").write(fixed_header.encode() + start_tag + compressed + tailer.encode())
def write_image_from_flex_array(out_image, pixel_values, header): '''Write a scaled CBF image from an array of pixel values and a header to add at the top. N.B. clobbers the binary size of the compressed data & the MD5 hash of the data.''' from cbflib_adaptbx import compress import binascii import os assert(not os.path.exists(out_image)) start_tag = binascii.unhexlify('0c1a04d5') compressed = compress(pixel_values) fixed_header = '' for record in header.split('\n')[:-1]: if 'X-Binary-Size:' in record: old_size = int(record.split()[-1]) fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed) elif 'Content-MD5' in record: pass else: fixed_header += '%s\n' % record tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n' open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def write_image_from_flex_array(out_image, pixel_values, header): '''Write a scaled CBF image from an array of pixel values and a header to add at the top. N.B. clobbers the binary size of the compressed data & the MD5 hash of the data.''' from cbflib_adaptbx import compress import binascii import os assert (not os.path.exists(out_image)) start_tag = binascii.unhexlify('0c1a04d5') compressed = compress(pixel_values) fixed_header = '' for record in header.split('\n')[:-1]: if 'X-Binary-Size:' in record: old_size = int(record.split()[-1]) fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed) elif 'Content-MD5' in record: pass else: fixed_header += '%s\n' % record tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n' open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def write_image_from_flex_array(out_image, pixel_values, header): """Write a scaled CBF image from an array of pixel values and a header to add at the top. N.B. clobbers the binary size of the compressed data & the MD5 hash of the data.""" from cbflib_adaptbx import compress assert not os.path.exists(out_image) start_tag = binascii.unhexlify("0c1a04d5") compressed = compress(pixel_values) fixed_header = "" header = header.decode() for record in header.split("\n")[:-1]: if "X-Binary-Size:" in record: fixed_header += f"X-Binary-Size: {len(compressed)}\r\n" elif "Content-MD5" in record: pass else: fixed_header += f"{record}\n" tailer = "\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n" open(out_image, "wb").write( fixed_header.encode() + start_tag + compressed + tailer.encode() )
def recompute_BKGINIT(bkginit_in, init_lp, bkginit_out): start_tag = binascii.unhexlify('0c1a04d5') data = open(bkginit_in, 'rb').read() data_offset = data.find(start_tag) + 4 cbf_header = data[:data_offset - 4] fast = 0 slow = 0 length = 0 for record in cbf_header.split('\n'): if 'X-Binary-Size-Fastest-Dimension' in record: fast = int(record.split()[-1]) elif 'X-Binary-Size-Second-Dimension' in record: slow = int(record.split()[-1]) elif 'X-Binary-Number-of-Elements' in record: length = int(record.split()[-1]) assert(length == fast * slow) pixel_values = uncompress(packed = data[data_offset:], fast = fast, slow = slow) untrusted = [] for record in open(init_lp): if 'UNTRUSTED_RECTANGLE=' in record: untrusted.append(map(int, record.replace('.', ' ').split()[1:5])) modified_pixel_values = copy.deepcopy(pixel_values) for s in range(5, slow - 5): y = s + 1 for f in range(5, fast - 5): x = f + 1 trusted = True for x0, x1, y0, y1 in untrusted: if (x >= x0) and (x <= x1) and (y >= y0) and (y <= y1): trusted = False break if trusted: pixel = pixel_values[s * fast + f] if pixel < 0: pixels = [] for j in range(-2, 3): for i in range(-2, 3): p = pixel_values[(s + j) * fast + f + i] if p > 0: pixels.append(p) modified_pixel_values[s * fast + f] = int( sum(pixels) / len(pixels)) open(bkginit_out, 'wb').write(cbf_header + start_tag + compress(modified_pixel_values)) return
def recompute_BKGINIT(bkginit_in, init_lp, bkginit_out): start_tag = binascii.unhexlify('0c1a04d5') data = open(bkginit_in, 'rb').read() data_offset = data.find(start_tag) + 4 cbf_header = data[:data_offset - 4] fast = 0 slow = 0 length = 0 for record in cbf_header.split('\n'): if 'X-Binary-Size-Fastest-Dimension' in record: fast = int(record.split()[-1]) elif 'X-Binary-Size-Second-Dimension' in record: slow = int(record.split()[-1]) elif 'X-Binary-Number-of-Elements' in record: length = int(record.split()[-1]) assert (length == fast * slow) pixel_values = uncompress(packed=data[data_offset:], fast=fast, slow=slow) untrusted = [] for record in open(init_lp): if 'UNTRUSTED_RECTANGLE=' in record: untrusted.append(map(int, record.replace('.', ' ').split()[1:5])) modified_pixel_values = copy.deepcopy(pixel_values) for s in range(5, slow - 5): y = s + 1 for f in range(5, fast - 5): x = f + 1 trusted = True for x0, x1, y0, y1 in untrusted: if (x >= x0) and (x <= x1) and (y >= y0) and (y <= y1): trusted = False break if trusted: pixel = pixel_values[s * fast + f] if pixel < 0: pixels = [] for j in range(-2, 3): for i in range(-2, 3): p = pixel_values[(s + j) * fast + f + i] if p > 0: pixels.append(p) modified_pixel_values[s * fast + f] = int( sum(pixels) / len(pixels)) open(bkginit_out, 'wb').write(cbf_header + start_tag + compress(modified_pixel_values)) return
def squishGain(cbf_file, out_name, force_gain=None): start_tag = binascii.unhexlify("0c1a04d5") data = cbf_file.read() data_offset = data.find(start_tag) + 4 cbf_header = data[:data_offset - 4] fast = 0 slow = 0 length = 0 for record in cbf_header.split("\n"): if "X-Binary-Size-Fastest-Dimension" in record: fast = int(record.split()[-1]) elif "X-Binary-Size-Second-Dimension" in record: slow = int(record.split()[-1]) elif "X-Binary-Size:" in record: xbsize_record = record length = int(record.split()[-1]) values = uncompress(packed=data[data_offset:data_offset + length], fast=fast, slow=slow) # remainder of the file, contains another CIF-BINARY-FORMAT-SECTION that looks # like just zero padding. tail = data[data_offset + length:] modified = copy.deepcopy(values) if force_gain: new_val = int(1000 * force_gain) else: # calculate the mean of values that are greater than zero. This is close # to 1000 times the "MEAN GAIN VALUE" reported in INIT.LP dval1d = modified.as_1d().as_double() mval = flex.mean(dval1d.select(dval1d > 0)) new_val = int(mval) # Set this value everywhere that is not a masked value marked by -3 print("Setting flat gain of {0}".format(new_val / 1000.0)) modified.set_selected(modified >= 0, new_val) # Compress the data compressed = compress(modified) nbytes = len(compressed) # Update the header pre, post = cbf_header.split(xbsize_record) new_xbsize_record = "X-Binary-Size:{0:10d}".format(nbytes) if xbsize_record.endswith("\r"): new_xbsize_record += "\r" new_cbf_header = pre + new_xbsize_record + post # Write the file out. open(out_name, "wb").write(new_cbf_header + start_tag + compressed + tail)
def save_cbf(im, orig_path, out_cbf=None, pOpt=False): ''' convert np.ndarray to cbf - im : np.ndarray or str : image to convert to cbf - orig_path : the path to exp data to get the header from those images - out_cbf : name of the .cbf output file - pOpt : Show cbf file with adxv if True (remember to set the path to adxv l.13) ''' from utils import glob_colors as colors if isinstance(im, str): im = np.load(im) if not out_cbf: out_cbf = in_npy.replace('.npy', '.cbf') orig_file = glob(orig_path + '*.cbf')[1] #;print(orig_file) # orig_file = "/home/tarik/Documents/data/ireloh/IRELOH_ED_Dataset_1/n14_a004_0484.cbf" start_tag = binascii.unhexlify("0c1a04d5") with open(orig_file, 'rb') as cbf: data = cbf.read() data_offset = data.find(start_tag) + 4 cbf_header = data[:data_offset - 4] fast = 0 slow = 0 length = 0 for record in cbf_header.decode().split("\n"): if "X-Binary-Size-Fastest-Dimension" in record: fast = int(record.split()[-1]) elif "X-Binary-Size-Second-Dimension" in record: slow = int(record.split()[-1]) elif "X-Binary-Size:" in record: xbsize_record = record length = int(record.split()[-1]) tail = data[data_offset + length:] # print(im.shape,fast,slow,length) im001 = flex.int32(im) #np.array(im,dtype=np.int32)) compressed = compress(im001) nbytes = len(compressed) # Update the header pre, post = cbf_header.decode().split(xbsize_record) new_xbsize_record = "X-Binary-Size:{0:10d}".format(nbytes) if xbsize_record.endswith("\r"): new_xbsize_record += "\r" new_cbf_header = pre + new_xbsize_record + post open(out_cbf, "wb").write(new_cbf_header.encode() + start_tag + compressed + tail) print(colors.green + 'file saved : \n' + colors.yellow + out_cbf + colors.black) if pOpt: p = Popen("%s %s" % (adxv, out_cbf), shell=True) p.wait()
def multiplyCounts(cbf_file, out_name, multiplier): start_tag = binascii.unhexlify("0c1a04d5") data = cbf_file.read() data_offset = data.find(start_tag) + 4 cbf_header = data[:data_offset - 4] fast = 0 slow = 0 length = 0 for record in cbf_header.split("\n"): if "X-Binary-Size-Fastest-Dimension" in record: fast = int(record.split()[-1]) elif "X-Binary-Size-Second-Dimension" in record: slow = int(record.split()[-1]) elif "X-Binary-Size:" in record: xbsize_record = record length = int(record.split()[-1]) values = uncompress(packed=data[data_offset:data_offset + length], fast=fast, slow=slow) # remainder of the file, contains another CIF-BINARY-FORMAT-SECTION that looks # like just zero padding. tail = data[data_offset + length:] # multiply all positive values modified = copy.deepcopy(values).as_1d() sel = modified > 0 new_val = modified.select(sel) * multiplier modified.set_selected(sel, new_val) # reshape modified.reshape(values.accessor()) # Compress the data compressed = compress(modified) nbytes = len(compressed) # Update the header pre, post = cbf_header.split(xbsize_record) new_xbsize_record = "X-Binary-Size:{0:10d}".format(nbytes) if xbsize_record.endswith("\r"): new_xbsize_record += "\r" new_cbf_header = pre + new_xbsize_record + post # Write the file out. open(out_name, "wb").write(new_cbf_header + start_tag + compressed + tail)
def add_zingers(imageset, params): from dxtbx.format.FormatCBF import FormatCBF assert issubclass(imageset.reader().get_format_class(), FormatCBF), "Only CBF format images supported" from cbflib_adaptbx import compress import binascii from numpy.random import poisson from random import sample import os.path for i in range(len(imageset)): image_data = imageset[i] num = poisson(params.zingers.average_per_image) index = sample(range(len(image_data)), num) value = list(poisson(params.zingers.average_intensity, num)) for j, v in zip(index, value): image_data[j] += v out_image = os.path.join(params.output.directory, "image_%04i.cbf" % i) start_tag = binascii.unhexlify("0c1a04d5") data = open(imageset.get_path(i), "rb").read() data_offset = data.find(start_tag) cbf_header = data[:data_offset] new_header = [] compressed = compress(image_data) old_size = 0 for record in cbf_header.split("\n")[:-1]: if "X-Binary-Size:" in record: old_size = int(record.split()[-1]) new_header.append("X-Binary-Size: %d\r\n" % len(compressed)) elif "Content-MD5" in record: pass else: new_header.append("%s\n" % record) tailer = data[data_offset + 4 + old_size:] with open(out_image, "wb") as f: f.write("".join(new_header) + start_tag + compressed + tailer) print("%s written with %d zingers" % (out_image, num)) return
def add_zingers(imageset, params): from dxtbx.format.FormatCBF import FormatCBF assert issubclass(imageset.reader().get_format_class(), FormatCBF), ( "Only CBF format images supported") from cbflib_adaptbx import compress import binascii from numpy.random import poisson from random import sample import os.path for i in range(len(imageset)): image_data = imageset[i] num = poisson(params.zingers.average_per_image) index = sample(range(len(image_data)), num) value = list(poisson(params.zingers.average_intensity, num)) for j, v in zip(index, value): image_data[j] += v out_image = os.path.join(params.output.directory, "image_%04i.cbf" % i) start_tag = binascii.unhexlify('0c1a04d5') data = open(imageset.get_path(i), 'rb').read() data_offset = data.find(start_tag) cbf_header = data[:data_offset] new_header = [] compressed = compress(image_data) old_size = 0 for record in cbf_header.split('\n')[:-1]: if 'X-Binary-Size:' in record: old_size = int(record.split()[-1]) new_header.append('X-Binary-Size: %d\r\n' % len(compressed)) elif 'Content-MD5' in record: pass else: new_header.append('%s\n' % record) tailer = data[data_offset + 4 + old_size:] with open(out_image, 'wb') as f: f.write(''.join(new_header) + start_tag + compressed + tailer) print '%s written with %d zingers' % (out_image, num) return
def basic_tests(verbose=True): initial_intdata = create_random_data_with_gaussian_distribution(0.0,100.0) #special deltas to test the compression algorithm addresses = [3,6,9,12,15,18] deltas = [-127,128,-32767,32768,-2147483647,2147483647] for x in xrange(6): initial_intdata[addresses[x]-1]=0 initial_intdata[addresses[x]]=deltas[x] if verbose: P=Profiler("compress") array_shape = initial_intdata.focus() if verbose: print array_shape compressed = compress(initial_intdata) if verbose: print len(compressed) if verbose: P=Profiler("uncompress") decompressed_dat = uncompress(packed=compressed, fast=array_shape[1], slow=array_shape[0]) if verbose: del P assert assert_equal(initial_intdata, decompressed_dat)
def write_image(out_image, pixel_values, header): from cbflib_adaptbx import compress assert not os.path.exists(out_image) start_tag = binascii.unhexlify("0c1a04d5") compressed = compress(pixel_values) fixed_header = "" for record in header.split("\n")[:-1]: if "X-Binary-Size:" in record: fixed_header += "X-Binary-Size: %d\r\n" % len(compressed) elif "Content-MD5" in record: pass else: fixed_header += "%s\n" % record tailer = "\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n" gz_open(out_image, "wb").write(fixed_header + start_tag + compressed + tailer)
def basic_tests(verbose=True): initial_intdata = create_random_data_with_gaussian_distribution(0.0, 100.0) #special deltas to test the compression algorithm addresses = [3, 6, 9, 12, 15, 18] deltas = [-127, 128, -32767, 32768, -2147483647, 2147483647] for x in range(6): initial_intdata[addresses[x] - 1] = 0 initial_intdata[addresses[x]] = deltas[x] if verbose: P = Profiler("compress") array_shape = initial_intdata.focus() if verbose: print(array_shape) compressed = compress(initial_intdata) if verbose: print(len(compressed)) if verbose: P = Profiler("uncompress") decompressed_dat = uncompress(packed=compressed, fast=array_shape[1], slow=array_shape[0]) if verbose: del P assert assert_equal(initial_intdata, decompressed_dat)
def write_image(out_image, pixel_values, header): from cbflib_adaptbx import compress import binascii import os assert(not os.path.exists(out_image)) start_tag = binascii.unhexlify('0c1a04d5') compressed = compress(pixel_values) fixed_header = '' for record in header.split('\n')[:-1]: if 'X-Binary-Size:' in record: old_size = int(record.split()[-1]) fixed_header += 'X-Binary-Size: %d\r\n' % len(compressed) elif 'Content-MD5' in record: pass else: fixed_header += '%s\n' % record tailer = '\r\n--CIF-BINARY-FORMAT-SECTION----\r\n;\r\n' gz_open(out_image, 'wb').write(fixed_header + start_tag + compressed + tailer)
def merge_cbf(imageset, n_images, out_prefix="sum_"): from dxtbx.format.FormatCBF import FormatCBF assert issubclass(imageset.get_format_class(), FormatCBF), ( "Only CBF format images supported") from cbflib_adaptbx import compress import binascii assert len(imageset) >= n_images n_output_images = len(imageset) // n_images in_oscillation = imageset.get_scan().get_oscillation()[1] out_oscillation = in_oscillation * n_images for i_out in range(n_output_images): data_out = None for j in range(n_images): i_in = (i_out*n_images) + j data_in = imageset.get_raw_data(i_in) assert len(data_in) == 1 data_in = data_in[0] if data_out is None: data_out = data_in else: # FIXME only add pixels to this which are > 0; image pixels < 0 # are meaningful and should be preserved; # Achieved by setting -ve values here to 0 before += # This assumes that -ve values are constant over all images data_special = data_in < 0 data_in.set_selected(data_special, 0) data_out += data_in out_image = "%s%04i.cbf" %(out_prefix, i_out+1) start_tag = binascii.unhexlify('0c1a04d5') data = open(imageset.get_path(i_out*n_images), 'rb').read() data_offset = data.find(start_tag) cbf_header = data[:data_offset] new_header = [] compressed = compress(data_out) old_size = 0 for record in cbf_header.split('\n')[:-1]: rsplit = record.split(' ') if 'X-Binary-Size:' in record: old_size = int(record.split()[-1]) new_header.append('X-Binary-Size: %d\r\n' % len(compressed)) elif 'Content-MD5' in record: pass elif len(rsplit) > 3 and rsplit[1] in { \ 'Exposure_time', 'Angle_increment', 'Exposure_period', 'Count_cutoff', \ 'Phi_increment', 'Omega_increment', 'Chi_increment' }: if rsplit[1] == 'Count_cutoff': # needs to be an integer new_header.append('%s\n' % ' '.join(rsplit[:2] + ['%d' % (n_images * int(rsplit[2]))] + rsplit[3:])) else: new_header.append('%s\n' % ' '.join(rsplit[:2] + ['%f' % (n_images * float(rsplit[2]))] + rsplit[3:])) else: new_header.append('%s\n' % record) loop_lines = [n for n, record in enumerate(new_header) if record.startswith('loop_')] multiply_fields = { '_diffrn_scan_axis.angle_range', '_diffrn_scan_axis.angle_increment', \ '_diffrn_scan_axis.displacement_range', '_diffrn_scan_axis.displacement_increment', \ '_diffrn_scan_frame.integration_time', '_diffrn_scan_frame.exposure_time', \ '_array_intensities.overload' } for loop_start in loop_lines: n = loop_start modifiers = [] while True: n = n + 1 line = new_header[n].strip() if line in { '', ';' }: # end of loop break elif line.startswith('_'): # loop header if line in multiply_fields: modifiers.append(n_images) else: modifiers.append(None) elif any(modifiers): # loop body # NOTE: This can break when fields are modified in loops with # 'Strings with spaces, as they are seen as multiple columns, or with' # _multiple _columns _defined _on _same _line _they _are _seen _as _one _column new_line = [ element if modifier is None else '%f' % (float(element) * modifier) \ for modifier, element in zip(modifiers, line.split()) ] new_header[n] = '%s\r\n' % ' '.join(new_line) tailer = data[data_offset + 4 + old_size:] with open(out_image, 'wb') as f: f.write(''.join(new_header) + start_tag + compressed + tailer) print '%s written' % out_image return
def merge_cbf(imageset, n_images, out_prefix="sum_", get_raw_data_from_imageset=True): from dxtbx.format.FormatCBF import FormatCBF assert issubclass( imageset.get_format_class(), FormatCBF ), "Only CBF format images supported" from cbflib_adaptbx import compress assert len(imageset) >= n_images n_output_images = len(imageset) // n_images n_digits = len(str(n_output_images)) for i_out in range(n_output_images): data_out = None for j in range(n_images): i_in = (i_out * n_images) + j if get_raw_data_from_imageset: data_in = imageset.get_raw_data(i_in) else: data_in = get_raw_data_from_file(imageset, i_in) assert len(data_in) == 1 data_in = data_in[0] if data_out is None: data_out = data_in else: # FIXME only add pixels to this which are > 0; image pixels < 0 # are meaningful and should be preserved; # Achieved by setting -ve values here to 0 before += # This assumes that -ve values are constant over all images data_special = data_in < 0 data_in.set_selected(data_special, 0) data_out += data_in out_image = "{prefix}{number:0{digits}d}.cbf".format( prefix=out_prefix, number=i_out + 1, digits=n_digits ) start_tag = binascii.unhexlify("0c1a04d5") with open(imageset.get_path(i_out * n_images), "rb") as fh: data = fh.read() data_offset = data.find(start_tag) cbf_header = data[:data_offset].decode("latin-1") new_header = [] compressed = compress(data_out) old_size = 0 for record in cbf_header.split("\n")[:-1]: rsplit = record.split(" ") if "X-Binary-Size:" in record: old_size = int(record.split()[-1]) new_header.append(f"X-Binary-Size: {len(compressed)}\r\n") elif "Content-MD5" in record: pass elif len(rsplit) > 3 and rsplit[1] in { "Exposure_time", "Angle_increment", "Exposure_period", "Count_cutoff", "Phi_increment", "Omega_increment", "Chi_increment", }: if rsplit[1] == "Count_cutoff": # needs to be an integer new_header.append( "%s\n" % " ".join( rsplit[:2] + ["%d" % (n_images * int(rsplit[2]))] + rsplit[3:] ) ) else: new_header.append( "%s\n" % " ".join( rsplit[:2] + ["%f" % (n_images * float(rsplit[2]))] + rsplit[3:] ) ) else: new_header.append(f"{record}\n") loop_lines = [ n for n, record in enumerate(new_header) if record.startswith("loop_") ] multiply_fields = { "_diffrn_scan_axis.angle_range", "_diffrn_scan_axis.angle_increment", "_diffrn_scan_axis.displacement_range", "_diffrn_scan_axis.displacement_increment", "_diffrn_scan_frame.integration_time", "_diffrn_scan_frame.exposure_time", "_array_intensities.overload", } for loop_start in loop_lines: n = loop_start modifiers = [] while True: n = n + 1 line = new_header[n].strip() if line in {"", ";"}: # end of loop break elif line.startswith("_"): # loop header if line in multiply_fields: modifiers.append(n_images) else: modifiers.append(None) elif any(modifiers): # loop body # NOTE: This can break when fields are modified in loops with # 'Strings with spaces, as they are seen as multiple columns, or with' # _multiple _columns _defined _on _same _line _they _are _seen _as _one _column new_line = [ element if modifier is None else f"{float(element) * modifier:f}" for modifier, element in zip(modifiers, line.split()) ] new_header[n] = f"{' '.join(new_line)}\r\n" tailer = data[data_offset + 4 + old_size :] with open(out_image, "wb") as f: f.write("".join(new_header).encode("latin-1")) f.write(start_tag) f.write(compressed) f.write(tailer) print(f"{out_image} written")
def make_cbf(in_name, template): f = h5py.File(in_name, "r") depends_on(f) mask = None start_tag = binascii.unhexlify("0c1a04d5") for j in range(len(f["/entry/sample/transformations/omega"][()])): block = 1 + (j // 1000) i = j % 1000 header = compute_cbf_header(f, j) depth, height, width = f["/entry/data/data_%06d" % block].shape data = flex.int(numpy.int32(f["/entry/data/data_%06d" % block][i])) good = data.as_1d() < 65535 data.as_1d().set_selected(~good, -2) # set the tile join regions to -1 - MOSFLM cares about this apparently if mask is None: mask = get_mask(width, height) data.as_1d().set_selected(mask.as_1d(), -1) compressed = compress(data) mime = """ _array_data.data ; --CIF-BINARY-FORMAT-SECTION-- Content-Type: application/octet-stream; conversions="x-CBF_BYTE_OFFSET" Content-Transfer-Encoding: BINARY X-Binary-Size: %d X-Binary-ID: 1 X-Binary-Element-Type: "signed 32-bit integer" X-Binary-Element-Byte-Order: LITTLE_ENDIAN X-Binary-Number-of-Elements: %d X-Binary-Size-Fastest-Dimension: %d X-Binary-Size-Second-Dimension: %d X-Binary-Size-Padding: 4095 """ % ( len(compressed), data.size(), data.focus()[1], data.focus()[0], ) padding = (bytearray(4095) + b"""--CIF-BINARY-FORMAT-SECTION---- ;""") with open(template % (j + 1), "wb") as fout: print(template % (j + 1)) fout.write( ("".join(header) + mime).replace("\n", "\r\n").encode("latin-1")) fout.write(start_tag) fout.write(compressed) fout.write(padding) f.close()
def merge_cbf(imageset, n_images, out_prefix="sum_"): from dxtbx.format.FormatCBF import FormatCBF assert issubclass(imageset.reader().get_format_class(), FormatCBF), ( "Only CBF format images supported") from cbflib_adaptbx import compress import binascii assert len(imageset) >= n_images n_output_images = len(imageset) // n_images in_oscillation = imageset.get_scan().get_oscillation()[1] out_oscillation = in_oscillation * n_images for i_out in range(n_output_images): data_out = None for j in range(n_images): i_in = (i_out*n_images) + j data_in = imageset.get_raw_data(i_in) assert len(data_in) == 1 data_in = data_in[0] if data_out is None: data_out = data_in else: # FIXME only add pixels to this which are > 0; image pixels < 0 # are meaningful and should be preserved; # Achieved by setting -ve values here to 0 before += # This assumes that -ve values are constant over all images data_special = data_in < 0 data_in.set_selected(data_special, 0) data_out += data_in out_image = "%s%04i.cbf" %(out_prefix, i_out+1) start_tag = binascii.unhexlify('0c1a04d5') data = open(imageset.get_path(i_out*n_images), 'rb').read() data_offset = data.find(start_tag) cbf_header = data[:data_offset] new_header = [] compressed = compress(data_out) old_size = 0 for record in cbf_header.split('\n')[:-1]: rsplit = record.split(' ') if 'X-Binary-Size:' in record: old_size = int(record.split()[-1]) new_header.append('X-Binary-Size: %d\r\n' % len(compressed)) elif 'Content-MD5' in record: pass elif len(rsplit) > 3 and rsplit[1] in { \ 'Exposure_time', 'Angle_increment', 'Exposure_period', 'Count_cutoff', \ 'Phi_increment', 'Omega_increment', 'Chi_increment' }: new_header.append('%s\n' % ' '.join(rsplit[:2] + ['%f' % (n_images * float(rsplit[2]))] + rsplit[3:])) else: new_header.append('%s\n' % record) loop_lines = [n for n, record in enumerate(new_header) if record.startswith('loop_')] multiply_fields = { '_diffrn_scan_axis.angle_range', '_diffrn_scan_axis.angle_increment', \ '_diffrn_scan_axis.displacement_range', '_diffrn_scan_axis.displacement_increment', \ '_diffrn_scan_frame.integration_time', '_diffrn_scan_frame.exposure_time', \ '_array_intensities.overload' } for loop_start in loop_lines: n = loop_start modifiers = [] while True: n = n + 1 line = new_header[n].strip() if line in { '', ';' }: # end of loop break elif line.startswith('_'): # loop header if line in multiply_fields: modifiers.append(n_images) else: modifiers.append(None) elif any(modifiers): # loop body # NOTE: This can break when fields are modified in loops with # 'Strings with spaces, as they are seen as multiple columns, or with' # _multiple _columns _defined _on _same _line _they _are _seen _as _one _column new_line = [ element if modifier is None else '%f' % (float(element) * modifier) \ for modifier, element in zip(modifiers, line.split()) ] new_header[n] = '%s\r\n' % ' '.join(new_line) tailer = data[data_offset + 4 + old_size:] with open(out_image, 'wb') as f: f.write(''.join(new_header) + start_tag + compressed + tailer) print '%s written' % out_image return