def read_xac_files(self, from_p1=False): op_to_p1 = None if from_p1: """ This option is currently for multi_determine_symmetry. Do not use this for ambiguity resolution! op_to_p1 is not considered when writing new HKL files. """ self.log_out.write("\nAveraging symmetry of all inputs..\n") cells = [] sgs = [] for f in self.xac_files: xac = XDS_ASCII(f, read_data=False) cells.append(xac.symm.unit_cell().parameters()) sgs.append(xac.symm.space_group()) assert len(set(sgs)) < 2 avg_symm = crystal.symmetry(list(numpy.median(cells, axis=0)), space_group=sgs[0]) op_to_p1 = avg_symm.change_of_basis_op_to_niggli_cell() self.log_out.write(" Averaged symmetry: %s (%s)\n" % (format_unit_cell(avg_symm.unit_cell()), sgs[0].info())) self.log_out.write(" Operator to Niggli cell: %s\n" % op_to_p1.as_hkl()) self.log_out.write(" Niggli cell: %s\n" % format_unit_cell(avg_symm.unit_cell().change_basis(op_to_p1))) print >>self.log_out, "\nReading" cells = [] bad_files, good_files = [], [] for i, f in enumerate(self.xac_files): print >>self.log_out, "%4d %s" % (i, f) xac = XDS_ASCII(f, i_only=True) self.log_out.write(" d_range: %6.2f - %5.2f" % xac.i_obs().resolution_range()) self.log_out.write(" n_ref=%6d" % xac.i_obs().size()) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=self.d_min) if self.min_ios is not None: a = a.select(a.data()/a.sigmas()>=self.min_ios) self.log_out.write(" n_ref_filtered=%6d" % a.size()) if from_p1: a = a.change_basis(op_to_p1).customized_copy(space_group_info=sgtbx.space_group_info("P1")) a = a.as_non_anomalous_array().merge_equivalents(use_internal_variance=False).array() self.log_out.write(" n_ref_merged=%6d\n" % a.size()) if a.size() < 2: self.log_out.write(" !! WARNING !! number of reflections is dangerously small!!\n") bad_files.append(f) else: self.arrays.append(a) cells.append(a.unit_cell().parameters()) good_files.append(f) if bad_files: self.xac_files = good_files self.bad_files = bad_files assert len(self.xac_files) == len(self.arrays) == len(cells) print >>self.log_out, "" self._representative_xs = crystal.symmetry(list(numpy.median(cells, axis=0)), space_group_info=self.arrays[0].space_group_info())
def calc_merging_stats(xac_file, cut_resolution=True): import iotbx.merging_statistics from yamtbx.dataproc.xds.xds_ascii import XDS_ASCII wdir = os.path.dirname(xac_file) pklout = os.path.join(wdir, "merging_stats.pkl") logout = open(os.path.join(wdir, "merging_stats.log"), "w") print >> logout, xac_file print >> logout, "" print >> logout, "Estimate cutoff" print >> logout, "================" obj = XDS_ASCII(xac_file, i_only=True) i_obs = obj.i_obs() d_min = None if i_obs.size() < 10: return try: cutoffs = resolution_cutoff.estimate_crude_resolution_cutoffs( i_obs=i_obs) cutoffs.show(out=logout) if cutoffs.cc_one_half_cut != float("inf") and cut_resolution: d_min = cutoffs.cc_one_half_cut except Sorry, e: print >> logout, e.message
def __init__(self, xac_files, d_min=3, min_ios=3, nproc=1, max_delta=3, log_out=null_out()): self.xac_files = xac_files self.log_out = log_out self.nproc = nproc self.arrays = [] self.max_delta = max_delta self.best_operators = None print >> self.log_out, "Reading" for i, f in enumerate(self.xac_files): print >> self.log_out, "%4d %s" % (i, f) xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=d_min) if min_ios is not None: a = a.select(a.data() / a.sigmas() >= min_ios) a = a.as_non_anomalous_array().merge_equivalents( use_internal_variance=False).array() self.arrays.append(a) print >> self.log_out, ""
def calc_merging_stats(xac_file, cut_resolution=True): import iotbx.merging_statistics from yamtbx.dataproc.xds.xds_ascii import XDS_ASCII wdir = os.path.dirname(xac_file) pklout = os.path.join(wdir, "merging_stats.pkl") logout = open(os.path.join(wdir, "merging_stats.log"), "w") print >>logout, xac_file print >>logout, "" print >>logout, "Estimate cutoff" print >>logout, "================" obj = XDS_ASCII(xac_file, i_only=True) i_obs = obj.i_obs() d_min = None if i_obs.size() < 10: return try: cutoffs = resolution_cutoff.estimate_crude_resolution_cutoffs(i_obs=i_obs) cutoffs.show(out=logout) if cutoffs.cc_one_half_cut != float("inf") and cut_resolution: d_min = cutoffs.cc_one_half_cut except Sorry, e: print >>logout, e.message
def run(hklin, hklin_merged=None, cone_angle=20., n_bins=10, anomalous=None, do_fit=True, log_out=null_out()): if 1: xac = XDS_ASCII(hklin, i_only=True) xac.remove_rejected() i_obs = xac.i_obs() #else: # import iotbx.mtz # i_obs = filter(lambda x: "SIGI" in x.info().label_string(), iotbx.mtz.object(hklin).as_miller_arrays(merge_equivalents=False))[0] print >> log_out, "Unmerged intensity read from", hklin i_obs.show_summary(log_out, prefix=" ") print >> log_out, "" if anomalous is not None and i_obs.anomalous_flag() != anomalous: print >> log_out, "Changing anomalous flag based on user's input" i_obs = i_obs.customized_copy(anomalous_flag=anomalous) if hklin_merged is not None: f = iotbx.file_reader.any_file(hklin) array_merged = f.file_server.get_xray_data(file_name=None, labels=None, ignore_all_zeros=True, parameter_scope="", prefer_anomalous=False, prefer_amplitudes=False) print >> log_out, "Merged intensity read from", hklin_merged array_merged.show_summary(log_out, prefix=" ") else: array_merged = i_obs.merge_equivalents( use_internal_variance=False).array() print >> log_out, "Merged intensity calculated" print >> log_out, "" bad_data = array_merged.select( array_merged.data() < -3 * array_merged.sigmas()) # FIXME What if already omitted.. i_obs = i_obs.delete_indices(other=bad_data) array_merged = array_merged.select(array_merged.sigmas() > 0) if anomalous is not None and not anomalous and array_merged.anomalous_flag( ): print >> log_out, "Converting to non-anomalous data..\n" array_merged = array_merged.average_bijvoet_mates() return make_aniso_stats_table(i_obs, array_merged, cone_angle, n_bins, do_fit, log_out)
def read_xac_files(xac_files, d_min=None, d_max=None, min_ios=None): arrays = collections.OrderedDict() for f in xac_files: xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=d_min, d_max=d_max) a = a.as_non_anomalous_array().merge_equivalents(use_internal_variance=False).array() if min_ios is not None: a = a.select(a.data()/a.sigmas()>=min_ios) arrays[f] = a return arrays
def read_xac_files(xac_files, d_min=None, d_max=None, min_ios=None): arrays = collections.OrderedDict() for f in xac_files: xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=d_min, d_max=d_max) a = a.as_non_anomalous_array().merge_equivalents( use_internal_variance=False).array() if min_ios is not None: a = a.select(a.data() / a.sigmas() >= min_ios) arrays[f] = a return arrays
def __init__(self, xac_files, d_min=3, min_ios=3, nproc=1, max_delta=3, log_out=null_out()): self.xac_files = xac_files self.log_out = log_out self.nproc = nproc self.arrays = [] self.max_delta = max_delta self.best_operators = None print >>self.log_out, "Reading" for i, f in enumerate(self.xac_files): print >>self.log_out, "%4d %s" % (i, f) xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=d_min) if min_ios is not None: a = a.select(a.data()/a.sigmas()>=min_ios) a = a.as_non_anomalous_array().merge_equivalents(use_internal_variance=False).array() self.arrays.append(a) print >>self.log_out, ""
ksb.assign_operators([debug_op, sgtbx.change_of_basis_op("h,k,l")]) print "right?:", [ i for i, x in enumerate(ksb.best_operators) if not x.is_identity_op() ] #ksb.debug_write_mtz() #ksb.modify_xds_ascii_files() quit() arrays = [] for f in xac_files: print "Reading", f xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=3) a = a.merge_equivalents(use_internal_variance=False).array() arrays.append(a) symm = arrays[0].crystal_symmetry() cosets = reindex.reindexing_operators(symm, symm) reidx_ops = cosets.combined_cb_ops() reidx_ops.sort(key=lambda x: not x.is_identity_op()) print " Possible reindex operators:", map(lambda x: str(x.as_hkl()), reidx_ops) determined = set([ 0, ]) old_ops = map(lambda x: 0, xrange(len(arrays)))
def run(lstin, params): xac_files = read_path_list(lstin) common0 = len(os.path.commonprefix(xac_files)) arrays = [] for f in xac_files: xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max) a = a.merge_equivalents(use_internal_variance=False).array() a = a.select(a.data() / a.sigmas() >= params.min_ios) arrays.append(a) # Prep args = [] for i in xrange(len(arrays) - 1): for j in xrange(i + 1, len(arrays)): args.append((i, j)) # Calc all CC worker = lambda x: calc_cc(arrays[x[0]], arrays[x[1]]) results = easy_mp.pool_map(fixed_func=worker, args=args, processes=params.nproc) # Make matrix mat = numpy.zeros(shape=(len(arrays), len(arrays))) for (i, j), (cc, nref) in zip(args, results): print j, i, cc mat[j, i] = cc open("%s.names" % params.prefix, "w").write("\n".join(map(lambda x: os.path.dirname(x[common0:]), xac_files))) open("%s.matrix" % params.prefix, "w").write(" ".join(map(lambda x: "%.4f" % x, mat.flatten()))) ofs = open("%s.dat" % params.prefix, "w") ofs.write("i j cc nref\n") for (i, j), (cc, nref) in zip(args, results): ofs.write("%4d %4d %.4f %4d\n" % (i, j, cc, nref)) open("%s_ana.R" % params.prefix, "w").write( """\ treeToList2 <- function(htree) { # stolen from $CCP4/share/blend/R/blend0.R groups <- list() itree <- dim(htree$merge)[1] for (i in 1:itree) { il <- htree$merge[i,1] ir <- htree$merge[i,2] if (il < 0) lab1 <- htree$labels[-il] if (ir < 0) lab2 <- htree$labels[-ir] if (il > 0) lab1 <- groups[[il]] if (ir > 0) lab2 <- groups[[ir]] lab <- c(lab1,lab2) lab <- as.integer(lab) groups <- c(groups,list(lab)) } return(groups) } cc<-scan("%s.matrix") md<-matrix(1-cc, ncol=%d, byrow=TRUE) labs<-read.table("%s.names") filenames<-read.table("%s")$V1 rownames(md)<-labs$V1 hc <- hclust(as.dist(md),method="ward") pdf("tree.pdf") plot(hc) dev.off() hc$labels <- 1:nrow(md) groups <- treeToList2(hc) cat("ClNumber Nds Clheight\\n",file="./CLUSTERS.txt") for (i in 1:length(groups)) { sorted_groups <- sort(groups[[i]]) linea <- paste(sprintf(" %%03d %%3d %%7.3f\\n", i,length(groups[[i]]),hc$height[i]),sep="") cat(linea, file="./CLUSTERS.txt", append=TRUE) write.table(filenames[sorted_groups], sprintf("cluster%%.3d.lst",i), quote=FALSE, row.names=FALSE, col.names=FALSE) } q(save="yes") """ % (params.prefix, len(arrays), params.prefix, lstin) ) print "R --vanilla < %s_ana.R" % params.prefix
print "altered:", idxes ksb.assign_operators([debug_op, sgtbx.change_of_basis_op("h,k,l")]) print "right?:", [i for i, x in enumerate(ksb.best_operators) if not x.is_identity_op()] #ksb.debug_write_mtz() #ksb.modify_xds_ascii_files() quit() arrays = [] for f in xac_files: print "Reading", f xac = XDS_ASCII(f, i_only=True) xac.remove_rejected() a = xac.i_obs().resolution_filter(d_min=3) a = a.merge_equivalents(use_internal_variance=False).array() arrays.append(a) symm = arrays[0].crystal_symmetry() cosets = reindex.reindexing_operators(symm, symm) reidx_ops = cosets.combined_cb_ops() reidx_ops.sort(key=lambda x: not x.is_identity_op()) print " Possible reindex operators:", map(lambda x: str(x.as_hkl()), reidx_ops) determined = set([0,]) old_ops = map(lambda x:0, xrange(len(arrays))) for ncycle in xrange(100): # max cycle new_ops = map(lambda x:0, xrange(len(arrays))) for i in xrange(len(arrays)):