def export_xds_ascii(integrated_data, experiment_list, hklout, summation=False, include_partials=False, keep_partials=False, var_model=(1,0)): '''Export data from integrated_data corresponding to experiment_list to an XDS_ASCII.HKL formatted text file.''' from dials.array_family import flex import math # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert(len(experiment_list) == 1) # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data['id'] >= 0) assert max(integrated_data['id']) == 0 if not summation: assert('intensity.prf.value' in integrated_data) if 'intensity.prf.variance' in integrated_data: selection = integrated_data.get_flags( integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags( integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data['intensity.sum.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d reflections with negative variance' % \ selection.count(True)) if 'intensity.prf.variance' in integrated_data: selection = integrated_data['intensity.prf.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d profile reflections with negative variance' % \ selection.count(True)) if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if 'partiality' in integrated_data: selection = integrated_data['partiality'] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info('Removing %d incomplete reflections' % \ selection.count(True)) experiment = experiment_list[0] # sort data before output nref = len(integrated_data['miller_index']) indices = flex.size_t_range(nref) import copy unique = copy.deepcopy(integrated_data['miller_index']) from cctbx.miller import map_to_asu map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) from scitbx import matrix from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame assert (not experiment.goniometer is None) unit_cell = experiment.crystal.get_unit_cell() from scitbx.array_family import flex from math import floor, sqrt assert(not experiment.scan is None) image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation(image_range[0]) # gather the required information for the reflection file nref = len(integrated_data['miller_index']) zdet = flex.double(integrated_data['xyzcal.px'].parts()[2]) miller_index = integrated_data['miller_index'] I = None sigI = None # export including scale factors if 'lp' in integrated_data: lp = integrated_data['lp'] else: lp = flex.double(nref, 1.0) if 'dqe' in integrated_data: dqe = integrated_data['dqe'] else: dqe = flex.double(nref, 1.0) scl = lp / dqe # profile correlation if 'profile.correlation' in integrated_data: prof_corr = 100.0 * integrated_data['profile.correlation'] else: prof_corr = flex.double(nref, 100.0) # partiality if 'partiality' in integrated_data: partiality = 100 * integrated_data['partiality'] else: prof_corr = flex.double(nref, 100.0) if summation: I = integrated_data['intensity.sum.value'] * scl V = integrated_data['intensity.sum.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data['intensity.prf.value'] * scl V = integrated_data['intensity.prf.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(hklout, 'w') # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1,0,0), slow, (0,1,0)) print 'Coordinate change:' print '%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n' % Rd.elems fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col( panel.get_normal())) orgx = - org.dot(fast) / qx orgy = - org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems axis = Rd * experiment.goniometer.get_rotation_axis() beam = Rd * experiment.beam.get_s0() cell_fmt = '%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f' axis_fmt = '%9.3f %9.3f %9.3f' fout.write('\n'.join([ '!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL\'S_LAW=TRUE', '!Generated by dials.export', '!DATA_RANGE= %d %d' % image_range, '!ROTATION_AXIS= %9.6f %9.6f %9.6f' % axis.elems, '!OSCILLATION_RANGE= %f' % phi_range, '!STARTING_ANGLE= %f' % phi_start, '!STARTING_FRAME= %d' % image_range[0], '!SPACE_GROUP_NUMBER= %d' % experiment.crystal.get_space_group().type().number(), '!UNIT_CELL_CONSTANTS= %s' % (cell_fmt % unit_cell.parameters()), '!UNIT_CELL_A-AXIS= %s' % (axis_fmt % real_space_ABC[0:3]), '!UNIT_CELL_B-AXIS= %s' % (axis_fmt % real_space_ABC[3:6]), '!UNIT_CELL_C-AXIS= %s' % (axis_fmt % real_space_ABC[6:9]), '!X-RAY_WAVELENGTH= %f' % experiment.beam.get_wavelength(), '!INCIDENT_BEAM_DIRECTION= %f %f %f' % beam.elems, '!NX= %d NY= %d QX= %f QY= %f' % (nx, ny, qx, qy), '!ORGX= %9.2f ORGY= %9.2f' % (orgx, orgy), '!DETECTOR_DISTANCE= %8.3f' % distance, '!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f' % fast.elems, '!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f' % slow.elems, '!VARIANCE_MODEL= %7.3e %7.3e' % var_model, '!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12', '!ITEM_H=1', '!ITEM_K=2', '!ITEM_L=3', '!ITEM_IOBS=4', '!ITEM_SIGMA(IOBS)=5', '!ITEM_XD=6', '!ITEM_YD=7', '!ITEM_ZD=8', '!ITEM_RLP=9', '!ITEM_PEAK=10', '!ITEM_CORR=11', '!ITEM_PSI=12', '!END_OF_HEADER', ''])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data['xyzcal.px'][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() f = (s - s0).normalize() # find component of beam perpendicular to f, e e = - (s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = (matrix.col(u).transpose() * UB.inverse()).normalize( ).transpose().rotate(axis, phi, deg=True) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write('%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n' % (h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi)) fout.write('!END_OF_DATA\n') fout.close() logger.info('Output %d reflections to %s' % (nref, hklout)) return
def export_sadabs(integrated_data, experiment_list, hklout, run=0, summation=False, include_partials=False, keep_partials=False, debug=False, predict=True): '''Export data from integrated_data corresponding to experiment_list to a file for input to SADABS. FIXME probably need to make a .p4p file as well...''' from dials.array_family import flex from scitbx import matrix import math # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert(len(experiment_list) == 1) # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data['id'] >= 0) assert max(integrated_data['id']) == 0 if not summation: assert('intensity.prf.value' in integrated_data) # strip out negative variance reflections: these should not really be there # FIXME Doing select on summation results. Should do on profile result if # present? Yes if 'intensity.prf.variance' in integrated_data: selection = integrated_data.get_flags( integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags( integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data['intensity.sum.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d reflections with negative variance' % \ selection.count(True)) if 'intensity.prf.variance' in integrated_data: selection = integrated_data['intensity.prf.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d profile reflections with negative variance' % \ selection.count(True)) if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if 'partiality' in integrated_data: selection = integrated_data['partiality'] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info('Removing %d incomplete reflections' % \ selection.count(True)) experiment = experiment_list[0] assert(not experiment.scan is None) # sort data before output nref = len(integrated_data['miller_index']) indices = flex.size_t_range(nref) perm = sorted(indices, key=lambda k: integrated_data['miller_index'][k]) integrated_data = integrated_data.select(flex.size_t(perm)) assert (not experiment.goniometer is None) axis = matrix.col(experiment.goniometer.get_rotation_axis_datum()) beam = matrix.col(experiment.beam.get_direction()) s0 = matrix.col(experiment.beam.get_s0()) F = matrix.sqr(experiment.goniometer.get_fixed_rotation()) S = matrix.sqr(experiment.goniometer.get_setting_rotation()) unit_cell = experiment.crystal.get_unit_cell() if debug: m_format = '%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f' c_format = '%.2f %.2f %.2f %.2f %.2f %.2f' logger.info('Unit cell parameters from experiment: %s' % (c_format % unit_cell.parameters())) logger.info('Symmetry: %s' % experiment.crystal.get_space_group().type( ).lookup_symbol()) logger.info('Goniometer fixed matrix:\n%s' % (m_format % F.elems)) logger.info('Goniometer setting matrix:\n%s' % (m_format % S.elems)) logger.info('Goniometer scan axis:\n%6.3f%6.3f%6.3f' % (axis.elems)) # detector scaling info assert(len(experiment.detector) == 1) panel = experiment.detector[0] dims = panel.get_image_size() pixel = panel.get_pixel_size() fast_axis = matrix.col(panel.get_fast_axis()) slow_axis = matrix.col(panel.get_slow_axis()) normal = fast_axis.cross(slow_axis) detector2t = s0.angle(normal, deg=True) origin = matrix.col(panel.get_origin()) if debug: logger.info('Detector fast, slow axes:') logger.info('%6.3f%6.3f%6.3f' % (fast_axis.elems)) logger.info('%6.3f%6.3f%6.3f' % (slow_axis.elems)) logger.info('Detector two theta (degrees): %.2f' % detector2t) scl_x = 512.0 / (dims[0] * pixel[0]) scl_y = 512.0 / (dims[1] * pixel[1]) image_range = experiment.scan.get_image_range() from cctbx.array_family import flex as cflex # implicit import from cctbx.miller import map_to_asu_isym # implicit import # gather the required information for the reflection file nref = len(integrated_data['miller_index']) zdet = flex.double(integrated_data['xyzcal.px'].parts()[2]) miller_index = integrated_data['miller_index'] I = None sigI = None # export including scale factors if 'lp' in integrated_data: lp = integrated_data['lp'] else: lp = flex.double(nref, 1.0) if 'dqe' in integrated_data: dqe = integrated_data['dqe'] else: dqe = flex.double(nref, 1.0) scl = lp / dqe if summation: I = integrated_data['intensity.sum.value'] * scl V = integrated_data['intensity.sum.variance'] * scl * scl assert V.all_gt(0) sigI = flex.sqrt(V) else: I = integrated_data['intensity.prf.value'] * scl V = integrated_data['intensity.prf.variance'] * scl * scl assert V.all_gt(0) sigI = flex.sqrt(V) # figure out scaling to make sure data fit into format 2F8.2 i.e. Imax < 1e5 Imax = flex.max(I) if debug: logger.info('Maximum intensity in file: %8.2f' % Imax) if Imax > 99999.0: scale = 99999.0 / Imax I = I * scale sigI = sigI * scale phi_start, phi_range = experiment.scan.get_image_oscillation(image_range[0]) if predict: logger.info('Using scan static predicted spot locations') from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor predictor = ScanStaticReflectionPredictor(experiment) UB = experiment.crystal.get_A() predictor.for_reflection_table(integrated_data, UB) if not experiment.crystal.num_scan_points: logger.info('No scan varying model: use static') static = True else: static = False fout = open(hklout, 'w') for j in range(nref): h, k, l = miller_index[j] if predict: x_mm, y_mm, z_rad = integrated_data['xyzcal.mm'][j] else: x_mm, y_mm, z_rad = integrated_data['xyzobs.mm.value'][j] z0 = integrated_data['xyzcal.px'][j][2] istol = int(round(10000 * unit_cell.stol((h, k, l)))) if predict or static: # work from a scan static model & assume perfect goniometer # FIXME maybe should work back in the option to predict spot positions UB = experiment.crystal.get_A() phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB else: # properly compute RUB for every reflection UB = experiment.crystal.get_A_at_scan_point(int(round(z0))) phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB x = RUB * (h, k, l) s = (s0 + x).normalize() # can also compute s based on centre of mass of spot # s = (origin + x_mm * fast_axis + y_mm * slow_axis).normalize() astar = (RUB * (1, 0, 0)).normalize() bstar = (RUB * (0, 1, 0)).normalize() cstar = (RUB * (0, 0, 1)).normalize() ix = beam.dot(astar) iy = beam.dot(bstar) iz = beam.dot(cstar) dx = s.dot(astar) dy = s.dot(bstar) dz = s.dot(cstar) x = x_mm * scl_x y = y_mm * scl_y z = (z_rad * 180 / math.pi - phi_start) / phi_range fout.write('%4d%4d%4d%8.2f%8.2f%4d%8.5f%8.5f%8.5f%8.5f%8.5f%8.5f' % \ (h, k, l, I[j], sigI[j], run, ix, dx, iy, dy, iz, dz)) fout.write('%7.2f%7.2f%8.2f%7.2f%5d\n' % (x, y, z, detector2t, istol)) fout.close() logger.info('Output %d reflections to %s' % (nref, hklout)) return
def export_xds_ascii(integrated_data, experiment_list, hklout, summation=False, include_partials=False, keep_partials=False, var_model=(1, 0)): '''Export data from integrated_data corresponding to experiment_list to an XDS_ASCII.HKL formatted text file.''' from dials.array_family import flex import math # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert (len(experiment_list) == 1) # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data['id'] >= 0) assert max(integrated_data['id']) == 0 if not summation: assert ('intensity.prf.value' in integrated_data) if 'intensity.prf.variance' in integrated_data: selection = integrated_data.get_flags(integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags( integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data['intensity.sum.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d reflections with negative variance' % \ selection.count(True)) if 'intensity.prf.variance' in integrated_data: selection = integrated_data['intensity.prf.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d profile reflections with negative variance' % \ selection.count(True)) if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if 'partiality' in integrated_data: selection = integrated_data['partiality'] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info('Removing %d incomplete reflections' % \ selection.count(True)) experiment = experiment_list[0] # sort data before output nref = len(integrated_data['miller_index']) indices = flex.size_t_range(nref) import copy unique = copy.deepcopy(integrated_data['miller_index']) from cctbx.miller import map_to_asu map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) from scitbx import matrix from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame assert (not experiment.goniometer is None) unit_cell = experiment.crystal.get_unit_cell() from scitbx.array_family import flex from math import floor, sqrt assert (not experiment.scan is None) image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) # gather the required information for the reflection file nref = len(integrated_data['miller_index']) zdet = flex.double(integrated_data['xyzcal.px'].parts()[2]) miller_index = integrated_data['miller_index'] I = None sigI = None # export including scale factors if 'lp' in integrated_data: lp = integrated_data['lp'] else: lp = flex.double(nref, 1.0) if 'dqe' in integrated_data: dqe = integrated_data['dqe'] else: dqe = flex.double(nref, 1.0) scl = lp / dqe # profile correlation if 'profile.correlation' in integrated_data: prof_corr = 100.0 * integrated_data['profile.correlation'] else: prof_corr = flex.double(nref, 100.0) # partiality if 'partiality' in integrated_data: partiality = 100 * integrated_data['partiality'] else: prof_corr = flex.double(nref, 100.0) if summation: I = integrated_data['intensity.sum.value'] * scl V = integrated_data['intensity.sum.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data['intensity.prf.value'] * scl V = integrated_data['intensity.prf.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(hklout, 'w') # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0)) print 'Coordinate change:' print '%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n' % Rd.elems fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(panel.get_normal())) orgx = -org.dot(fast) / qx orgy = -org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems axis = Rd * experiment.goniometer.get_rotation_axis() beam = Rd * experiment.beam.get_s0() cell_fmt = '%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f' axis_fmt = '%9.3f %9.3f %9.3f' fout.write('\n'.join([ '!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL\'S_LAW=TRUE', '!Generated by dials.export', '!DATA_RANGE= %d %d' % image_range, '!ROTATION_AXIS= %9.6f %9.6f %9.6f' % axis.elems, '!OSCILLATION_RANGE= %f' % phi_range, '!STARTING_ANGLE= %f' % phi_start, '!STARTING_FRAME= %d' % image_range[0], '!SPACE_GROUP_NUMBER= %d' % experiment.crystal.get_space_group().type().number(), '!UNIT_CELL_CONSTANTS= %s' % (cell_fmt % unit_cell.parameters()), '!UNIT_CELL_A-AXIS= %s' % (axis_fmt % real_space_ABC[0:3]), '!UNIT_CELL_B-AXIS= %s' % (axis_fmt % real_space_ABC[3:6]), '!UNIT_CELL_C-AXIS= %s' % (axis_fmt % real_space_ABC[6:9]), '!X-RAY_WAVELENGTH= %f' % experiment.beam.get_wavelength(), '!INCIDENT_BEAM_DIRECTION= %f %f %f' % beam.elems, '!NX= %d NY= %d QX= %f QY= %f' % (nx, ny, qx, qy), '!ORGX= %9.2f ORGY= %9.2f' % (orgx, orgy), '!DETECTOR_DISTANCE= %8.3f' % distance, '!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f' % fast.elems, '!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f' % slow.elems, '!VARIANCE_MODEL= %7.3e %7.3e' % var_model, '!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12', '!ITEM_H=1', '!ITEM_K=2', '!ITEM_L=3', '!ITEM_IOBS=4', '!ITEM_SIGMA(IOBS)=5', '!ITEM_XD=6', '!ITEM_YD=7', '!ITEM_ZD=8', '!ITEM_RLP=9', '!ITEM_PEAK=10', '!ITEM_CORR=11', '!ITEM_PSI=12', '!END_OF_HEADER', '' ])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data['xyzcal.px'][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() f = (s - s0).normalize() # find component of beam perpendicular to f, e e = -(s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = (matrix.col(u).transpose() * UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write('%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n' % (h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi)) fout.write('!END_OF_DATA\n') fout.close() logger.info('Output %d reflections to %s' % (nref, hklout)) return