def find_unconnected_xds_files(xds_files, min_ios=3, d_min=None): import networkx as nx from yamtbx.dataproc.xds import xds_ascii from yamtbx.dataproc.xds import integrate_hkl_as_flex if len(xds_files) < 2: return [] G = nx.Graph() arrays = [] for f in xds_files: if xds_ascii.is_xds_ascii(f): a = xds_ascii.XDS_ASCII(f, i_only=True).i_obs().resolution_filter(d_min=d_min) elif integrate_hkl_as_flex.is_integrate_hkl(f): a = integrate_hkl_as_flex.reader(f, ["IOBS", "SIGMA"]).i_obs().resolution_filter(d_min=d_min) else: raise "Never reaches here" a = a.select(a.sigmas() > 0) a = a.merge_equivalents(use_internal_variance=False).array() arrays.append(a.select(a.data() / a.sigmas() >= min_ios)) for i in xrange(len(arrays) - 1): for j in xrange(i + 1, len(arrays)): matchs = arrays[i].match_indices(other=arrays[j], assert_is_similar_symmetry=False) if matchs.pairs().size() >= 10: G.add_edge(i, j) print "edge", i, j ccomps = map(lambda x: x, nx.connected_components(G)) print "DEBUG:: Connected components=", ccomps keep_idxes = ccomps[0] remove_idxes = filter(lambda x: x not in keep_idxes, xrange(len(xds_files))) return remove_idxes
def run(files, params): print "filename", for cut in params.cut_ios: print "cut_ios_%.2f" % cut, print for f in files: is_xac = xds_ascii.is_xds_ascii(f) i_obs = None if is_xac: xac = xds_ascii.XDS_ASCII(f, read_data=True, i_only=True) xac.remove_rejected() i_obs = xac.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max) if params.fix_variance_model: ao, bo = xac.variance_model an, bn = params.variance_model i_obs = i_obs.customized_copy(sigmas = flex.sqrt(flex.abs(an * (i_obs.sigmas()**2/ao + (bn-bo)*flex.pow2(i_obs.data()))))) else: ihkl = integrate_hkl_as_flex.reader(f, read_columns=("IOBS","SIGMA")) i_obs = ihkl.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max) if params.fix_variance_model: a, b = params.variance_model i_obs = i_obs.customized_copy(sigmas = flex.sqrt(a * (i_obs.sigmas()**2 + b*flex.pow2(i_obs.data())))) cutoffs = eval_resolution(i_obs, params.n_bins, params.cut_ios) print "%s %s" % (f, " ".join(map(lambda x: "%.2f"%x, cutoffs)))
def import_integrated(integrate_hkl, min_ios=3): reader = integrate_hkl_as_flex.reader(integrate_hkl, "IOBS,SIGMA,XCAL,YCAL,ZCAL,XOBS,YOBS,ZOBS,ISEG".split(",")) # reference: dials/command_line/import_xds.py table = flex.reflection_table() table["id"] = flex.int(len(reader.hkl), 0) table["panel"] = flex.size_t(len(reader.hkl), 0) # only assuming single panel table["miller_index"] = reader.hkl table["xyzcal.px.value"] = flex.vec3_double(reader.data["XCAL"], reader.data["YCAL"], reader.data["ZCAL"]) table["xyzobs.px.value"] = flex.vec3_double(reader.data["XOBS"], reader.data["YOBS"], reader.data["ZOBS"]) table["intensity.cor.value"] = reader.data["IOBS"] table["intensity.cor.sigma"] = reader.data["SIGMA"] # not valid name, just for making selection table["flags"] = flex.size_t(len(table), table.flags.indexed | table.flags.strong) table = table.select(table["intensity.cor.sigma"] > 0) table = table.select(table["intensity.cor.value"]/table["intensity.cor.sigma"] >= min_ios) table = table.select(table["xyzobs.px.value"].norms() > 0) # remove invalid xyzobs # dummy table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1,1,1)) # TODO appropriate variance value table["s1"] = flex.vec3_double(len(table), (0,0,0)) # will be updated by set_obs_s1() del table["intensity.cor.value"] del table["intensity.cor.sigma"] return table
def find_unconnected_xds_files(xds_files, min_ios=3, d_min=None): import networkx as nx from yamtbx.dataproc.xds import xds_ascii from yamtbx.dataproc.xds import integrate_hkl_as_flex if len(xds_files) < 2: return [] G = nx.Graph() arrays = [] for f in xds_files: if xds_ascii.is_xds_ascii(f): a = xds_ascii.XDS_ASCII( f, i_only=True).i_obs().resolution_filter(d_min=d_min) elif integrate_hkl_as_flex.is_integrate_hkl(f): a = integrate_hkl_as_flex.reader( f, ["IOBS", "SIGMA"]).i_obs().resolution_filter(d_min=d_min) else: raise "Never reaches here" a = a.select(a.sigmas() > 0) a = a.merge_equivalents(use_internal_variance=False).array() arrays.append(a.select(a.data() / a.sigmas() >= min_ios)) for i in xrange(len(arrays) - 1): for j in xrange(i + 1, len(arrays)): matchs = arrays[i].match_indices(other=arrays[j], assert_is_similar_symmetry=False) if matchs.pairs().size() >= 10: G.add_edge(i, j) print "edge", i, j ccomps = map(lambda x: x, nx.connected_components(G)) print "DEBUG:: Connected components=", ccomps keep_idxes = ccomps[0] remove_idxes = filter(lambda x: x not in keep_idxes, xrange(len(xds_files))) return remove_idxes
def run(params, out): print >> out, "Frames:", params.frames backup_needed = xds_files.generated_by_DEFPIX + ( "XDS.INP", "BKGINIT.cbf", ) bk_prefix = make_backup(backup_needed, wdir=params.xdsdir) ret = {} # {frame: [matches, spots, predicted]} try: # run DEFPIX to limit resolution. modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"), [("JOB", "DEFPIX"), ("INCLUDE_RESOLUTION_RANGE", "50 %.2f" % params.d_min)]) p = subprocess.Popen("xds", cwd=params.xdsdir) p.wait() # copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT) shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"), os.path.join(params.xdsdir, "BKGINIT.cbf")) for frame in params.frames: print >> out, "Frame %d" % frame print >> out, "====================\n" # search spots if params.spotfinder == "xds": spotxds = get_colspot_result(frame_ranges=[ [frame, frame], ], wdir=params.xdsdir) spots = map(lambda x: x[:2], spotxds.collected_spots(with_resolution=False)) else: raise "Sorry!" # run INTEGRATE to collect predicted coords integrate_results = xds_predict_mitai.run( param_source=os.path.join(params.xdsdir, "XPARM.XDS"), frame_num=frame, wdir=params.xdsdir, need_adx=False, sigmar=params.sigmar, sigmab=params.sigmab) # read predicted coords tmp = filter(lambda x: x.endswith(".HKL"), integrate_results) if len(tmp) == 0: print >> out, "Integration failed!" ret[frame] = (0, len(spots), 0) continue integrate_hkl = tmp[0] cols = integrate_hkl_as_flex.reader(integrate_hkl, [], False).get_column_names() i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL") predicted = [] for l in open(integrate_hkl): if l.startswith("!"): continue sp = l.split() predicted.append(map(float, (sp[i_xcal], sp[i_ycal]))) # compare them nmatch = calc_matches( spots, predicted, params.distance_limit_in_px, open( os.path.join(params.xdsdir, "matched_predicted_%.4d.adx" % frame), "w")) #nmatch = calc_matches(predicted, spots, params.distance_limit_in_px, # open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w")) ret[frame] = (nmatch, len(spots), len(predicted)) finally: revert_files(backup_needed, bk_prefix, wdir=params.xdsdir) print >> out for frame in sorted(ret): nmatch, nspots, npredicted = ret[frame] print >> out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % ( frame, nmatch, npredicted, 100. * float(nmatch) / npredicted if npredicted > 0 else float("nan")) print >> out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % ( frame, nmatch, nspots, 100. * float(nmatch) / nspots if nspots > 0 else float("nan")) print >> out return ret
def run(params, log_out): xa = XDS_ASCII(params.xds_ascii, log_out) rejected_array = miller.array(miller_set=miller.set(crystal_symmetry=xa.symm, indices=xa.indices, anomalous_flag=False), data=xa.sigma_iobs < 0) xa_zd = miller.array(miller_set=miller.set(crystal_symmetry=xa.symm, indices=xa.indices, anomalous_flag=False), data=xa.zd) # Read ZCAL, not ZOBS, because ZOBS (and XOBS, YOBS) can be zero (in case unobserved). integ_data = integrate_hkl_as_flex.reader(params.integrate_hkl, ["MAXC","ZCAL"]).arrays() maxc_array, integ_zcal = integ_data["MAXC"], integ_data["ZCAL"] assert integ_zcal.unit_cell().is_similar_to(xa_zd.unit_cell()) # two set of indices should be comparable. overload_flags = maxc_array.customized_copy(data=maxc_array.data() == params.overload) print "Overloaded observations in INTEGRATE.HKL:", overload_flags.data().count(True) print "Rejected (sigma<0) observations in XDS_ASCII.HKL:", rejected_array.data().count(True) # common_sets() does not work correctly for unmerged data! rejected_zd = xa_zd.select(rejected_array.data()) #reject_indices = flex.bool([False for i in xrange(overload_flags.size())]) print "making indices..........." import yamtbx_utils_ext integ_zcal = integ_zcal.sort(by_value="packed_indices") # Must be sorted before C++ function below!! reject_indices = yamtbx_utils_ext.make_selection_for_xds_unmerged(rejected_zd.indices(), rejected_zd.data(), integ_zcal.indices(), integ_zcal.data(), 3.) """ # This loop is too slow! for i in xrange(rejected_zd.size()): sel = integ_zcal.indices() == rejected_zd.indices()[i] sel &= (integ_zcal.data() - rejected_zd.data()[i]) < 3 reject_indices.set_selected(sel, True) print i, rejected_zd.size(), sel.count(True) """ """ # This loop is also too slow! for j in xrange(integ_zcal.size()): # j: INTEGRATE.HKL if rejected_zd.indices()[i] != integ_zcal.indices()[j]: continue if abs(rejected_zd.data()[i] - integ_zcal.data()[j]) < 3: # within 3 frames.. OK? reject_indices[j] = True """ print "Found rejected observations in INTEGRATE.HKL:", reject_indices.count(True) overload_flags.data().set_selected(reject_indices, False) # Set 'Un-overloaded' print "Remained overloaded observations:", overload_flags.data().count(True) overload_flags_partial = overload_flags.map_to_asu().merge_equivalents(incompatible_flags_replacement=True).array() overload_flags_all = overload_flags.map_to_asu().merge_equivalents(incompatible_flags_replacement=False).array() mtz_object = iotbx.mtz.object(params.hklin). \ add_crystal("crystal", "project", overload_flags_all.unit_cell()). \ add_dataset(name="dataset", wavelength=0). \ add_miller_array(miller_array=overload_flags_all, column_root_label="SATURATED_ALL"). \ add_miller_array(miller_array=overload_flags_partial, column_root_label="SATURATED_PART"). \ mtz_object() mtz_object.write(file_name=params.hklout)
def xds_sequence(img_in, topdir, data_root_dir, params): relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir) workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0])) print workdir frame_num = None if not os.path.exists(img_in): if "<>" in img_in: img_in, num = img_in.split("<>") frame_num = int(num) if not os.path.exists(img_in): print "Error: Not found!!", img_in return workdir += "_%.6d" % frame_num assert img_in.endswith(".h5") else: for ext in (".bz2", ".gz", ".xz"): if os.path.exists(img_in+ext): img_in += ext break if params.tmpdir is not None: tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir) else: tmpdir = workdir if not os.path.exists(tmpdir): os.makedirs(tmpdir) xparm = os.path.join(tmpdir, "XPARM.XDS") xdsinp = os.path.join(tmpdir, "XDS.INP") integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL") decilog = open(os.path.join(tmpdir, "decision.log"), "w") try: print >>decilog, "Paramters:" libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ") print >>decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S") # Prepare XDS.INP if params.sfx.cheetah_mpccd: xdsinp_str = sfx_xds_inp(img_in, xdsinp, params) else: if frame_num is not None: # Eiger from yamtbx.dataproc import eiger img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf") eiger.extract_to_minicbf(img_in, frame_num, img_in_work) else: ext = os.path.splitext(img_in)[1] img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext) os.symlink(img_in, img_in_work) xdsinp_str = xds_inp.generate_xds_inp(img_files=[img_in_work], inp_dir=tmpdir, reverse_phi=True, anomalous=params.anomalous, spot_range=None, minimum=False, crystal_symmetry=None, integrate_nimages=None, osc_range=params.osc_range, orgx=params.orgx, orgy=params.orgy, rotation_axis=params.rotation_axis, distance=params.distance) ofs = open(xdsinp, "w") ofs.write(xdsinp_str) ofs.close() # Start processing modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF"), ("MAXIMUM_NUMBER_OF_PROCESSORS", "1"), ("MINPK", "%.2f"%params.minpk), ("PROFILE_FITTING", bool_to_str(params.profile_fitting)), ("STRONG_PIXEL", "%.2f"%params.strong_pixel), ("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d"%params.minimum_number_of_pixels_in_a_spot), ("SEPMIN", "%.2f"%params.sepmin), ("CLUSTER_RADIUS", "%.2f"%params.cluster_radius), ("INDEX_ERROR", "%.4f"%params.index_error), ("INDEX_MAGNITUDE", "%d"%params.index_magnitude), ("INDEX_QUALITY", "%.2f"%params.index_quality), ("REFINE(IDXREF)", " ".join(params.refine_idxref)), ("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)), ("VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels)) ]) if len(params.extra_inp_str) > 0: ofs = open(xdsinp, "a") ofs.write("\n%s\n" % "\n".join(params.extra_inp_str)) ofs.close() run_xds(wdir=tmpdir, show_progress=False) if params.tryhard: try_indexing_hard(tmpdir, show_progress=True, decilog=decilog) # If Cell hint exists, try to use it.. if params.sgnum > 0: flag_try_cell_hint = False if not os.path.isfile(xparm): flag_try_cell_hint = True else: xsxds = XPARM(xparm).crystal_symmetry() cosets = check_cell(params, xsxds) if cosets.double_cosets is None: flag_try_cell_hint = True if flag_try_cell_hint: print >>decilog, " Worth trying to use prior cell for indexing." modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell))), ("SPACE_GROUP_NUMBER", "%d"%params.sgnum), ]) run_xds(wdir=tmpdir, show_progress=False) modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"), ]) if not os.path.isfile(xparm): raise ProcFailed("Indexing failed") if params.checkcell.check and params.sgnum > 0: xsxds = XPARM(xparm).crystal_symmetry() cosets = check_cell(params, xsxds) if cosets.double_cosets is None: raise ProcFailed("Incompatible cell. Indexing failed.") if not cosets.combined_cb_ops()[0].is_identity_op(): print >>decilog, "Re-idxref to match reference symmetry." xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK?? modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"), ("SPACE_GROUP_NUMBER", "%d"%params.sgnum), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters()))) ]) run_xds(wdir=tmpdir, show_progress=False) modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min)), ]) # To Integration modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),]) run_xds(wdir=tmpdir, show_progress=False) if not os.path.isfile(integrate_hkl): raise ProcFailed("Integration failed.") # Determine unit cell in CORRECT if params.refine_correct: tmp = [("REFINE(CORRECT)", "ALL"), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell)))] else: # XXX What if CELL is refined in INTEGRATE? xsxds = XPARM(xparm).crystal_symmetry() cosets = check_cell(params, xsxds) if cosets.double_cosets is None: raise ProcFailed(" Incompatible cell. Failed before CORRECT.") xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK?? tmp = [("REFINE(CORRECT)", ""), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))] # PEAK-corrected INTEGRATE.HKL ihk = os.path.join(tmpdir, "INTEGRATE.HKL") ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL") ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL") inhkl = integrate_hkl_as_flex.reader(ihk, [], False) inhkl.write_peak_corrected(ihk_part) os.rename(ihk, ihk_full) modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"), ("DATA_RANGE", "1 20000"), ("CORRECTIONS", ""), ("NBATCH", "1"), ("SPACE_GROUP_NUMBER", "%d"%params.sgnum)] + tmp) xac = os.path.join(tmpdir, "XDS_ASCII.HKL") xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL") xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL") # CORRECT for full os.symlink(ihk_full, ihk) run_xds(wdir=tmpdir, comm="xds", show_progress=False) if os.path.isfile(xac): os.rename(xac, xac_full) os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_full.LP")) os.remove(ihk) # CORRECT for part os.symlink(ihk_part, ihk) run_xds(wdir=tmpdir, comm="xds", show_progress=False) if os.path.isfile(xac): os.rename(xac, xac_part) os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_part.LP")) os.remove(ihk) if params.pickle_hkl: for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)): print >>decilog, "Pickling %s" % os.path.basename(f) x = xds_ascii.XDS_ASCII(f, log_out=decilog) if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster pickle.dump(x, open(f+".pkl", "w"), -1) if params.pickle_hkl: for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)): print >>decilog, "Pickling %s" % os.path.basename(f) inhkl = integrate_hkl_as_flex.reader(f, read_columns=["IOBS","SIGMA","XCAL","YCAL","RLP","PEAK","MAXC"]) pickle.dump(inhkl, open(f+".pkl", "w"), -1) except ProcFailed, e: print >>decilog, "Processing failed: %s" % e.message
def run(params, out): print >>out, "Frames:", params.frames backup_needed = xds_files.generated_by_DEFPIX + ("XDS.INP","BKGINIT.cbf",) bk_prefix = make_backup(backup_needed, wdir=params.xdsdir) ret = {} # {frame: [matches, spots, predicted]} try: # run DEFPIX to limit resolution. modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"), [("JOB", "DEFPIX"), ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%params.d_min)]) p = subprocess.Popen("xds", cwd=params.xdsdir) p.wait() # copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT) shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"), os.path.join(params.xdsdir, "BKGINIT.cbf")) for frame in params.frames: print >>out, "Frame %d" % frame print >>out, "====================\n" # search spots if params.spotfinder == "xds": spotxds = get_colspot_result(frame_ranges=[[frame, frame],], wdir=params.xdsdir) spots = map(lambda x: x[:2], spotxds.collected_spots(with_resolution=False)) else: raise "Sorry!" # run INTEGRATE to collect predicted coords integrate_results = xds_predict_mitai.run(param_source=os.path.join(params.xdsdir, "XPARM.XDS"), frame_num=frame, wdir=params.xdsdir, need_adx=False, sigmar=params.sigmar, sigmab=params.sigmab) # read predicted coords tmp = filter(lambda x:x.endswith(".HKL"), integrate_results) if len(tmp) == 0: print >>out, "Integration failed!" ret[frame] = (0, len(spots), 0) continue integrate_hkl = tmp[0] cols = integrate_hkl_as_flex.reader(integrate_hkl, [], False).get_column_names() i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL") predicted = [] for l in open(integrate_hkl): if l.startswith("!"): continue sp = l.split() predicted.append(map(float, (sp[i_xcal], sp[i_ycal]))) # compare them nmatch = calc_matches(spots, predicted, params.distance_limit_in_px, open(os.path.join(params.xdsdir, "matched_predicted_%.4d.adx"%frame), "w")) #nmatch = calc_matches(predicted, spots, params.distance_limit_in_px, # open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w")) ret[frame] = (nmatch, len(spots), len(predicted)) finally: revert_files(backup_needed, bk_prefix, wdir=params.xdsdir) print >>out for frame in sorted(ret): nmatch, nspots, npredicted = ret[frame] print >>out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (frame, nmatch, npredicted, 100.*float(nmatch)/npredicted if npredicted>0 else float("nan")) print >>out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (frame, nmatch, nspots, 100.*float(nmatch)/nspots if nspots>0 else float("nan")) print >>out return ret
def run(params, log_out): xa = XDS_ASCII(params.xds_ascii, log_out) rejected_array = miller.array(miller_set=miller.set( crystal_symmetry=xa.symm, indices=xa.indices, anomalous_flag=False), data=xa.sigma_iobs < 0) xa_zd = miller.array(miller_set=miller.set(crystal_symmetry=xa.symm, indices=xa.indices, anomalous_flag=False), data=xa.zd) # Read ZCAL, not ZOBS, because ZOBS (and XOBS, YOBS) can be zero (in case unobserved). integ_data = integrate_hkl_as_flex.reader(params.integrate_hkl, ["MAXC", "ZCAL"]).arrays() maxc_array, integ_zcal = integ_data["MAXC"], integ_data["ZCAL"] assert integ_zcal.unit_cell().is_similar_to( xa_zd.unit_cell()) # two set of indices should be comparable. overload_flags = maxc_array.customized_copy( data=maxc_array.data() == params.overload) print "Overloaded observations in INTEGRATE.HKL:", overload_flags.data( ).count(True) print "Rejected (sigma<0) observations in XDS_ASCII.HKL:", rejected_array.data( ).count(True) # common_sets() does not work correctly for unmerged data! rejected_zd = xa_zd.select(rejected_array.data()) #reject_indices = flex.bool([False for i in xrange(overload_flags.size())]) print "making indices..........." import yamtbx_utils_ext integ_zcal = integ_zcal.sort( by_value="packed_indices" ) # Must be sorted before C++ function below!! reject_indices = yamtbx_utils_ext.make_selection_for_xds_unmerged( rejected_zd.indices(), rejected_zd.data(), integ_zcal.indices(), integ_zcal.data(), 3.) """ # This loop is too slow! for i in xrange(rejected_zd.size()): sel = integ_zcal.indices() == rejected_zd.indices()[i] sel &= (integ_zcal.data() - rejected_zd.data()[i]) < 3 reject_indices.set_selected(sel, True) print i, rejected_zd.size(), sel.count(True) """ """ # This loop is also too slow! for j in xrange(integ_zcal.size()): # j: INTEGRATE.HKL if rejected_zd.indices()[i] != integ_zcal.indices()[j]: continue if abs(rejected_zd.data()[i] - integ_zcal.data()[j]) < 3: # within 3 frames.. OK? reject_indices[j] = True """ print "Found rejected observations in INTEGRATE.HKL:", reject_indices.count( True) overload_flags.data().set_selected(reject_indices, False) # Set 'Un-overloaded' print "Remained overloaded observations:", overload_flags.data().count( True) overload_flags_partial = overload_flags.map_to_asu().merge_equivalents( incompatible_flags_replacement=True).array() overload_flags_all = overload_flags.map_to_asu().merge_equivalents( incompatible_flags_replacement=False).array() mtz_object = iotbx.mtz.object(params.hklin). \ add_crystal("crystal", "project", overload_flags_all.unit_cell()). \ add_dataset(name="dataset", wavelength=0). \ add_miller_array(miller_array=overload_flags_all, column_root_label="SATURATED_ALL"). \ add_miller_array(miller_array=overload_flags_partial, column_root_label="SATURATED_PART"). \ mtz_object() mtz_object.write(file_name=params.hklout)
def xds_sequence(img_in, topdir, data_root_dir, params): relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir) workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0])) print workdir frame_num = None if not os.path.exists(img_in): if "<>" in img_in: img_in, num = img_in.split("<>") frame_num = int(num) if not os.path.exists(img_in): print "Error: Not found!!", img_in return workdir += "_%.6d" % frame_num assert img_in.endswith(".h5") else: for ext in (".bz2", ".gz", ".xz"): if os.path.exists(img_in + ext): img_in += ext break if params.tmpdir is not None: tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir) else: tmpdir = workdir if not os.path.exists(tmpdir): os.makedirs(tmpdir) xparm = os.path.join(tmpdir, "XPARM.XDS") xdsinp = os.path.join(tmpdir, "XDS.INP") integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL") decilog = open(os.path.join(tmpdir, "decision.log"), "w") try: print >> decilog, "Paramters:" libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ") print >> decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S") # Prepare XDS.INP if params.sfx.cheetah_mpccd: xdsinp_str = sfx_xds_inp(img_in, xdsinp, params) else: if frame_num is not None: # Eiger from yamtbx.dataproc import eiger img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf") eiger.extract_to_minicbf(img_in, frame_num, img_in_work) else: ext = os.path.splitext(img_in)[1] img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext) os.symlink(img_in, img_in_work) xdsinp_str = xds_inp.generate_xds_inp( img_files=[img_in_work], inp_dir=tmpdir, reverse_phi=True, anomalous=params.anomalous, spot_range=None, minimum=False, crystal_symmetry=None, integrate_nimages=None, osc_range=params.osc_range, orgx=params.orgx, orgy=params.orgy, rotation_axis=params.rotation_axis, distance=params.distance, ) ofs = open(xdsinp, "w") ofs.write(xdsinp_str) ofs.close() # Start processing modify_xdsinp( xdsinp, inp_params=[ ("JOB", "XYCORR INIT COLSPOT IDXREF"), ("MAXIMUM_NUMBER_OF_PROCESSORS", "1"), ("MINPK", "%.2f" % params.minpk), ("PROFILE_FITTING", bool_to_str(params.profile_fitting)), ("STRONG_PIXEL", "%.2f" % params.strong_pixel), ("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d" % params.minimum_number_of_pixels_in_a_spot), ("SEPMIN", "%.2f" % params.sepmin), ("CLUSTER_RADIUS", "%.2f" % params.cluster_radius), ("INDEX_ERROR", "%.4f" % params.index_error), ("INDEX_MAGNITUDE", "%d" % params.index_magnitude), ("INDEX_QUALITY", "%.2f" % params.index_quality), ("REFINE(IDXREF)", " ".join(params.refine_idxref)), ("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)), ( "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels), ), ], ) if len(params.extra_inp_str) > 0: ofs = open(xdsinp, "a") ofs.write("\n%s\n" % "\n".join(params.extra_inp_str)) ofs.close() run_xds(wdir=tmpdir, show_progress=False) if params.tryhard: try_indexing_hard(tmpdir, show_progress=True, decilog=decilog) # If Cell hint exists, try to use it.. if params.sgnum > 0: flag_try_cell_hint = False if not os.path.isfile(xparm): flag_try_cell_hint = True else: xsxds = XPARM(xparm).crystal_symmetry() cosets = check_cell(params, xsxds) if cosets.double_cosets is None: flag_try_cell_hint = True if flag_try_cell_hint: print >> decilog, " Worth trying to use prior cell for indexing." modify_xdsinp( xdsinp, inp_params=[ ("JOB", "IDXREF"), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))), ("SPACE_GROUP_NUMBER", "%d" % params.sgnum), ], ) run_xds(wdir=tmpdir, show_progress=False) modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0")]) if not os.path.isfile(xparm): raise ProcFailed("Indexing failed") if params.checkcell.check and params.sgnum > 0: xsxds = XPARM(xparm).crystal_symmetry() cosets = check_cell(params, xsxds) if cosets.double_cosets is None: raise ProcFailed("Incompatible cell. Indexing failed.") if not cosets.combined_cb_ops()[0].is_identity_op(): print >> decilog, "Re-idxref to match reference symmetry." xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK?? modify_xdsinp( xdsinp, inp_params=[ ("JOB", "IDXREF"), ("SPACE_GROUP_NUMBER", "%d" % params.sgnum), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))), ], ) run_xds(wdir=tmpdir, show_progress=False) modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min))]) # To Integration modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE")]) run_xds(wdir=tmpdir, show_progress=False) if not os.path.isfile(integrate_hkl): raise ProcFailed("Integration failed.") # Determine unit cell in CORRECT if params.refine_correct: tmp = [ ("REFINE(CORRECT)", "ALL"), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))), ] else: # XXX What if CELL is refined in INTEGRATE? xsxds = XPARM(xparm).crystal_symmetry() cosets = check_cell(params, xsxds) if cosets.double_cosets is None: raise ProcFailed(" Incompatible cell. Failed before CORRECT.") xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK?? tmp = [ ("REFINE(CORRECT)", ""), ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))), ] # PEAK-corrected INTEGRATE.HKL ihk = os.path.join(tmpdir, "INTEGRATE.HKL") ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL") ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL") inhkl = integrate_hkl_as_flex.reader(ihk, [], False) inhkl.write_peak_corrected(ihk_part) os.rename(ihk, ihk_full) modify_xdsinp( xdsinp, inp_params=[ ("JOB", "CORRECT"), ("DATA_RANGE", "1 20000"), ("CORRECTIONS", ""), ("NBATCH", "1"), ("SPACE_GROUP_NUMBER", "%d" % params.sgnum), ] + tmp, ) xac = os.path.join(tmpdir, "XDS_ASCII.HKL") xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL") xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL") # CORRECT for full os.symlink(ihk_full, ihk) run_xds(wdir=tmpdir, comm="xds", show_progress=False) if os.path.isfile(xac): os.rename(xac, xac_full) os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_full.LP")) os.remove(ihk) # CORRECT for part os.symlink(ihk_part, ihk) run_xds(wdir=tmpdir, comm="xds", show_progress=False) if os.path.isfile(xac): os.rename(xac, xac_part) os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_part.LP")) os.remove(ihk) if params.pickle_hkl: for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)): print >> decilog, "Pickling %s" % os.path.basename(f) x = xds_ascii.XDS_ASCII(f, log_out=decilog) if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster pickle.dump(x, open(f + ".pkl", "w"), -1) if params.pickle_hkl: for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)): print >> decilog, "Pickling %s" % os.path.basename(f) inhkl = integrate_hkl_as_flex.reader( f, read_columns=["IOBS", "SIGMA", "XCAL", "YCAL", "RLP", "PEAK", "MAXC"] ) pickle.dump(inhkl, open(f + ".pkl", "w"), -1) except ProcFailed, e: print >> decilog, "Processing failed: %s" % e.message