def run_cycles(self, xds_files): self.removed_files = [] self.removed_reason = {} print >> self.out, "********************* START FUNCTION ***********************" # Remove unconnected files here. remove_idxes = find_unconnected_xds_files(xds_files, min_ios=3, d_min=self.d_min) print >> self.out, "DEBUG:: Need to remove unconnected %d files" % len(remove_idxes) for i in sorted(remove_idxes): print >> self.out, " %.3d %s" % (i + 1, xds_files[i]) self.removed_files.append(xds_files[i]) self.removed_reason[xds_files[i]] = "no_common_refls" keep_idxes = filter(lambda x: x not in remove_idxes, xrange(len(xds_files))) self.run_cycle(map(lambda i: xds_files[i], keep_idxes)) # Final cycle; average cells and merge again. used_files = set(xds_files).difference(set(self.removed_files)) sg, cell = self.average_cells(used_files) print >> self.out, "Final scaling with averaged cell." print >> self.out, "Averaged cell= %s" % (cell) # call mtzutils to change cell! mtz_finalcell = "pointless_finalcell.mtz" util.call( cmd="mtzutils hklin %s hklout %s" % (os.path.relpath(self.mtzin, self.workdir), mtz_finalcell), wdir=self.workdir, expects_in=[os.path.relpath(self.mtzin, self.workdir)], expects_out=[mtz_finalcell], stdin="cell %s\nend\n" % cell, stdout=open(os.path.join(self.workdir, "mtzutils.log"), "w"), ) inp_str = "" for i, f in enumerate(used_files): brange = self.batch_info[f] inp_str += "RUN %3d BATCH %4d to %4d\n" % (i + 1, brange[0], brange[1]) aimless.run_aimless( mtzin=mtz_finalcell, wdir=self.workdir, anomalous=self.anomalous_flag, d_min=self.d_min, prefix="aimless", add_stdin=inp_str, ) ctruncate_arg = "-hklin aimless.mtz -hklout ctruncate.mtz -colin '/*/*/[IMEAN,SIGIMEAN]'" if self.anomalous_flag: ctruncate_arg += " -colano '/*/*/[I(+),SIGI(+),I(-),SIGI(-)]'" util.call( cmd="ctruncate %s" % ctruncate_arg, wdir=self.workdir, expects_in=["aimless.mtz"], expects_out=["ctruncate.mtz"], stdout=open(os.path.join(self.workdir, "ctruncate.log"), "w"), ) return self.removed_files, self.removed_reason
def run_cycles(self, xds_files): self.removed_files = [] self.removed_reason = {} print >> self.out, "********************* START FUNCTION ***********************" # Remove unconnected files here. remove_idxes = find_unconnected_xds_files(xds_files, min_ios=3, d_min=self.d_min) print >> self.out, "DEBUG:: Need to remove unconnected %d files" % len( remove_idxes) for i in sorted(remove_idxes): print >> self.out, " %.3d %s" % (i + 1, xds_files[i]) self.removed_files.append(xds_files[i]) self.removed_reason[xds_files[i]] = "no_common_refls" keep_idxes = filter(lambda x: x not in remove_idxes, xrange(len(xds_files))) self.run_cycle(map(lambda i: xds_files[i], keep_idxes)) # Final cycle; average cells and merge again. used_files = set(xds_files).difference(set(self.removed_files)) sg, cell = self.average_cells(used_files) print >> self.out, "Final scaling with averaged cell." print >> self.out, "Averaged cell= %s" % (cell) # call mtzutils to change cell! mtz_finalcell = "pointless_finalcell.mtz" util.call(cmd="mtzutils hklin %s hklout %s" % (os.path.relpath(self.mtzin, self.workdir), mtz_finalcell), wdir=self.workdir, expects_in=[os.path.relpath(self.mtzin, self.workdir)], expects_out=[mtz_finalcell], stdin="cell %s\nend\n" % cell, stdout=open(os.path.join(self.workdir, "mtzutils.log"), "w")) inp_str = "" for i, f in enumerate(used_files): brange = self.batch_info[f] inp_str += "RUN %3d BATCH %4d to %4d\n" % (i + 1, brange[0], brange[1]) aimless.run_aimless(mtzin=mtz_finalcell, wdir=self.workdir, anomalous=self.anomalous_flag, d_min=self.d_min, prefix="aimless", add_stdin=inp_str) ctruncate_arg = "-hklin aimless.mtz -hklout ctruncate.mtz -colin '/*/*/[IMEAN,SIGIMEAN]'" if self.anomalous_flag: ctruncate_arg += " -colano '/*/*/[I(+),SIGI(+),I(-),SIGI(-)]'" util.call(cmd="ctruncate %s" % ctruncate_arg, wdir=self.workdir, expects_in=["aimless.mtz"], expects_out=["ctruncate.mtz"], stdout=open(os.path.join(self.workdir, "ctruncate.log"), "w")) return self.removed_files, self.removed_reason
def run_cycle(self, xds_files, do_rejection=True): if len(xds_files) == 0: print >> self.out, "Error: no files given." return inp_str = "" for i, f in enumerate(xds_files): brange = self.batch_info[f] inp_str += "RUN %3d BATCH %4d to %4d\n" % (i + 1, brange[0], brange[1]) print >> self.out, "DEBUG:: running aimless with %3d files.." % len(xds_files) aimless.run_aimless( mtzin=os.path.relpath(self.mtzin, self.workdir), wdir=self.workdir, anomalous=self.anomalous_flag, d_min=self.d_min, prefix="aimless", add_stdin=inp_str, ) aimless_log = os.path.join(self.workdir, "aimless.log") # XXX Aimless error handling here. if not do_rejection: return # Remove bad data remove_idxes = [] if self.reject_method == "delta_cc1/2": print >> self.out, "Rejection based on delta_CC1/2 in %s shell" % self.delta_cchalf_bin table = aimless.read_summary(aimless_log) i_stat = 0 if self.delta_cchalf_bin == "total" else 2 prev_cchalf = table["cc_half"][i_stat] prev_nuniq = table["nuniq"][i_stat] # file_name->idx table remaining_files = collections.OrderedDict(map(lambda x: x[::-1], enumerate(xds_files))) for i in xrange(len(xds_files) - 1): # if only one file, cannot proceed. tmpdir = os.path.join(self.workdir, "reject_test_%.3d" % i) cchalf_list = aimless.calc_cchalf_by_removing( wdir=tmpdir, mtzin=self.mtzin, batch_info=self.batch_info, inpfiles=remaining_files.keys(), anomalous_flag=self.anomalous_flag, d_min=self.d_min, stat_bin=self.delta_cchalf_bin, nproc=self.nproc, nproc_each=self.nproc_each, batchjobs=self.batchjobs, ) rem_idx, cc_i, nuniq_i = cchalf_list[0] # First (largest) is worst one to remove. rem_idx_in_org = remaining_files[remaining_files.keys()[rem_idx]] # Decision making by CC1/2 print >> self.out, "DEBUG:: remove %3d if %.4f*%d > %.4f*%d" % ( rem_idx_in_org, cc_i, nuniq_i, prev_cchalf, prev_nuniq, ) if cc_i * nuniq_i <= prev_cchalf * prev_nuniq: break print >> self.out, "Removing idx= %3d gains CC1/2 by %.4f" % (rem_idx_in_org, cc_i - prev_cchalf) prev_cchalf, prev_nuniq = cc_i, nuniq_i remove_idxes.append(rem_idx_in_org) del remaining_files[remaining_files.keys()[rem_idx]] # remove file from table else: print >> self.out, "ERROR:: Unsupported reject_method (%s)" % reject_method if len(remove_idxes) > 0: print >> self.out, "DEBUG:: Need to remove %d files" % len(remove_idxes) for i in sorted(remove_idxes): print >> self.out, " %.3d %s" % (i + 1, xds_files[i]) self.removed_files.append(xds_files[i]) self.removed_reason[xds_files[i]] = "badcc" if self.next_delta_cchalf_bin != []: self.delta_cchalf_bin = self.next_delta_cchalf_bin.pop(0) do_rejection = True else: do_rejection = False if do_rejection or len(remove_idxes) > 0: keep_idxes = filter(lambda x: x not in remove_idxes, xrange(len(xds_files))) self.workdir = self.request_next_workdir() self.run_cycle(map(lambda i: xds_files[i], keep_idxes), do_rejection=do_rejection)
def run_cycle(self, xds_files, do_rejection=True): if len(xds_files) == 0: print >> self.out, "Error: no files given." return inp_str = "" for i, f in enumerate(xds_files): brange = self.batch_info[f] inp_str += "RUN %3d BATCH %4d to %4d\n" % (i + 1, brange[0], brange[1]) print >> self.out, "DEBUG:: running aimless with %3d files.." % len( xds_files) aimless.run_aimless(mtzin=os.path.relpath(self.mtzin, self.workdir), wdir=self.workdir, anomalous=self.anomalous_flag, d_min=self.d_min, prefix="aimless", add_stdin=inp_str) aimless_log = os.path.join(self.workdir, "aimless.log") # XXX Aimless error handling here. if not do_rejection: return # Remove bad data remove_idxes = [] if self.reject_method == "delta_cc1/2": print >> self.out, "Rejection based on delta_CC1/2 in %s shell" % self.delta_cchalf_bin table = aimless.read_summary(aimless_log) i_stat = 0 if self.delta_cchalf_bin == "total" else 2 prev_cchalf = table["cc_half"][i_stat] prev_nuniq = table["nuniq"][i_stat] # file_name->idx table remaining_files = collections.OrderedDict( map(lambda x: x[::-1], enumerate(xds_files))) for i in xrange(len(xds_files) - 1): # if only one file, cannot proceed. tmpdir = os.path.join(self.workdir, "reject_test_%.3d" % i) cchalf_list = aimless.calc_cchalf_by_removing( wdir=tmpdir, mtzin=self.mtzin, batch_info=self.batch_info, inpfiles=remaining_files.keys(), anomalous_flag=self.anomalous_flag, d_min=self.d_min, stat_bin=self.delta_cchalf_bin, nproc=self.nproc, nproc_each=self.nproc_each, batchjobs=self.batchjobs) rem_idx, cc_i, nuniq_i = cchalf_list[ 0] # First (largest) is worst one to remove. rem_idx_in_org = remaining_files[remaining_files.keys() [rem_idx]] # Decision making by CC1/2 print >> self.out, "DEBUG:: remove %3d if %.4f*%d > %.4f*%d" % ( rem_idx_in_org, cc_i, nuniq_i, prev_cchalf, prev_nuniq) if cc_i * nuniq_i <= prev_cchalf * prev_nuniq: break print >> self.out, "Removing idx= %3d gains CC1/2 by %.4f" % ( rem_idx_in_org, cc_i - prev_cchalf) prev_cchalf, prev_nuniq = cc_i, nuniq_i remove_idxes.append(rem_idx_in_org) del remaining_files[remaining_files.keys() [rem_idx]] # remove file from table else: print >> self.out, "ERROR:: Unsupported reject_method (%s)" % reject_method if len(remove_idxes) > 0: print >> self.out, "DEBUG:: Need to remove %d files" % len( remove_idxes) for i in sorted(remove_idxes): print >> self.out, " %.3d %s" % (i + 1, xds_files[i]) self.removed_files.append(xds_files[i]) self.removed_reason[xds_files[i]] = "badcc" if self.next_delta_cchalf_bin != []: self.delta_cchalf_bin = self.next_delta_cchalf_bin.pop(0) do_rejection = True else: do_rejection = False if do_rejection or len(remove_idxes) > 0: keep_idxes = filter(lambda x: x not in remove_idxes, xrange(len(xds_files))) self.workdir = self.request_next_workdir() self.run_cycle(map(lambda i: xds_files[i], keep_idxes), do_rejection=do_rejection)