def run_cycles(self, xds_ascii_files):
self.all_data_root = os.path.dirname(os.path.commonprefix(xds_ascii_files))
self.removed_files = []
self.removed_reason = {}
print >>self.out, "********************* START FUNCTION ***********************"
if self.reference_file:
self.run_cycle([self.reference_file,]+xds_ascii_files)
else:
self.run_cycle(xds_ascii_files)
if self.res_params.estimate:
#self.cut_resolution(self.get_last_cycle_number())
for run_i in xrange(1, self.get_last_cycle_number()+1):
try: self.estimate_resolution(run_i)
except: print >>self.out, traceback.format_exc() # Don't want to stop the program.
for wd in glob.glob(os.path.join(self.workdir_org, "run_*")):
if os.path.exists(os.path.join(wd, "ccp4")): continue
xscale_hkl = os.path.abspath(os.path.join(wd, "xscale.hkl"))
sg = None # Use user-specified one. Otherwise follow pointless.
try:
sg = XDS_ASCII(xscale_hkl, read_data=False).symm.space_group()
laue_symm_str = str(sg.build_derived_reflection_intensity_group(False).info())
worker = Pointless()
result = worker.run_for_symm(xdsin=xscale_hkl,
logout=os.path.join(wd, "pointless.log"),
choose_laue=laue_symm_str,
xdsin_to_p1=True)
if "symm" in result:
print >>self.out, "Pointless suggestion (forcing %s symmetry):" % laue_symm_str
result["symm"].show_summary(self.out, " ")
sg = str(result["symm"].space_group_info())
else:
print >>self.out, "Pointless failed."
except:
# Don't want to stop the program.
print >>self.out, traceback.format_exc()
if self.space_group is not None:
sg = str(self.space_group.info())
try:
xds2mtz.xds2mtz(xds_file=xscale_hkl,
dir_name=os.path.join(wd, "ccp4"),
run_xtriage=True, run_ctruncate=True,
with_multiplicity=True,
space_group=sg,
flag_source=self.ref_mtz)
except:
# Don't want to stop the program.
print >>self.out, traceback.format_exc()
return self.removed_files, self.removed_reason
def run(params, out=sys.stdout):
cm = CheckMulti(topdir=params.topdir, xdsdirs=params.xdsdir, out=out)
cm.get_symms_from_xds_results()
cm.construct_graph(params.tol_length, params.tol_angle)
cm.group_xds_results()
print
ret = cm.grouped_dirs
if len(ret) == 0:
return ret
print >>out
print >>out, "About the largest group:"
for idx, wd in enumerate(ret[0]):
gxparm_xds = os.path.join(wd, "GXPARM.XDS")
correct_lp = os.path.join(wd, "CORRECT.LP")
print >>out, "%.3d %s" % (idx, os.path.relpath(wd, params.topdir) if params.topdir is not None else wd),
if not os.path.isfile(gxparm_xds):
print >>out, "Unsuccessful"
continue
sg = sgtbx.space_group_info(XPARM(gxparm_xds).spacegroup)
clp = correctlp.CorrectLp(correct_lp)
if "all" in clp.table:
cmpl = clp.table["all"]["cmpl"][-1]
else:
cmpl = float("nan")
ISa = clp.a_b_ISa[-1]
print >>out, "%10s ISa=%5.2f Cmpl=%5.1f " % (sg, ISa, cmpl)
if params.do_pointless:
worker = Pointless()
files = map(lambda x: os.path.join(x, "INTEGRATE.HKL"), ret[0])
#print files
files = filter(lambda x: os.path.isfile(x), files)
print >>out, "\nRunning pointless for the largest member."
result = worker.run_for_symm(xdsin=files,
logout="pointless.log",
tolerance=10, d_min=5)
if "symm" in result:
print >>out, " pointless suggested", result["symm"].space_group_info()
if 0:
import pylab
pos = nx.spring_layout(G)
#pos = nx.spectral_layout(G)
#pos = nx.circular_layout(G)
#nx.draw_networkx_nodes(G, pos, node_size = 100, nodelist=others, node_color = 'w')
nx.draw_networkx_nodes(G, pos, node_size = 100, node_color = 'w')
nx.draw_networkx_edges(G, pos, width = 1)
nx.draw_networkx_labels(G, pos, font_size = 12, font_family = 'sans-serif', font_color = 'r')
pylab.xticks([])
pylab.yticks([])
pylab.savefig("network.png")
pylab.show()
return cm
def merge_datasets(params, workdir, xds_files, cells, space_group):
if not os.path.exists(workdir): os.makedirs(workdir)
out = open(os.path.join(workdir, "merge.log"), "w")
if params.program == "xscale":
cycles = multi_merging.xscale.XscaleCycles(
workdir,
anomalous_flag=params.anomalous,
d_min=params.d_min,
d_max=params.d_max,
reject_method=params.reject_method,
reject_params=params.rejection,
xscale_params=params.xscale,
res_params=params.resolution,
reference_file=params.reference_file,
space_group=space_group,
ref_mtz=params.reference.data
if params.reference.copy_test_flag else None,
out=out,
nproc=params.nproc,
batch_params=params.batch)
unused_files, reasons = cycles.run_cycles(xds_files)
used_files = set(xds_files).difference(set(unused_files))
print >> out
print >> out, " SUMMARY "
print >> out, "========================"
for i, files in enumerate((used_files, unused_files)):
print >> out, "\n%6s %4d files:\n" % (
("Used", "Unused")[i], len(files))
if len(files) == 0:
continue
maxlen_f = max(
map(lambda f: len(os.path.relpath(f, params.workdir)), files))
for f in files:
cell = cells[f]
merge_log = os.path.join(os.path.dirname(f),
"merging_stats.log")
try:
lines = open(merge_log).readlines()
resn = float(
filter(lambda x: x.startswith("Resolution:"),
lines)[0].split()[-1])
cmpl = float(
filter(lambda x: x.startswith("Completeness:"),
lines)[0].split()[-1].replace("%", ""))
except:
resn = float("nan")
cmpl = float("nan")
if i == 1: # print reason
print >> out, "%-15s" % reasons.get(f, "unknown"),
print >> out, ("%-" + str(maxlen_f) + "s") % os.path.relpath(
f, params.workdir), cell,
#print >>out, "ISa=%5.1f" % correctlp.get_ISa(os.path.join(os.path.dirname(f), "CORRECT.LP")),
print >> out, "Cmpl=%3.0f%%, Resn= %.1f" % (cmpl, resn)
ret = []
tkvals = lambda x: (x[-1], x[0], x[-2]) # overall, inner, outer
for i in xrange(1, cycles.get_last_cycle_number() + 1):
wd = os.path.join(workdir, "run_%.2d" % i)
xscale_lp = os.path.join(wd, "XSCALE.LP")
table = xscalelp.read_stats_table(xscale_lp)
num_files = len(xscalelp.get_read_data(xscale_lp))
xtriage_logfile = os.path.join(wd, "ccp4", "logfile.log")
aniso = xds_aniso_analysis.parse_logfile(
os.path.join(wd, "aniso.log"))
cellinfo = cycles.cell_info_at_cycles[i]
ret.append([
i, wd, num_files,
dict(cmpl=tkvals(table["cmpl"]),
redundancy=tkvals(table["redundancy"]),
i_over_sigma=tkvals(table["i_over_sigma"]),
r_meas=tkvals(table["r_meas"]),
cc_half=tkvals(table["cc_half"]),
sig_ano=tkvals(table["sig_ano"]),
cc_ano=tkvals(table["cc_ano"]),
drange=tkvals(table["d_range"]),
lp=xscale_lp,
xtriage_log=xtriage.XtriageLogfile(xtriage_logfile),
aniso=aniso,
lcv=cellinfo[1],
alcv=cellinfo[2],
dmin_est=cycles.dmin_est_at_cycles.get(i, float("nan")))
])
xscale_lp = os.path.join(cycles.current_working_dir(), "XSCALE.LP")
print >> out, "\nFinal statistics:\n"
print >> out, xscalelp.snip_stats_table(xscale_lp)
return ret
elif params.program == "aimless":
worker = Pointless()
print >> out, "\nRunning pointless"
runinfo = worker.run_copy(hklout="pointless.mtz",
wdir=workdir,
xdsin=xds_files,
logout=os.path.join(workdir,
"pointless.log"),
tolerance=30)
# Table of file name -> Batch range
assert len(xds_files) == len(runinfo)
batch_info = collections.OrderedDict(
map(lambda x: (x[0], (x[1][1:3])), zip(xds_files, runinfo)))
cycles = multi_merging.aimless.AimlessCycles(
workdir,
anomalous_flag=params.anomalous,
d_min=params.d_min,
d_max=params.d_max,
reject_method=params.reject_method,
cc_cutoff=params.rejection.lpstats.pwcc.abs_cutoff,
delta_cchalf_bin=params.rejection.delta_cchalf.bin,
mtzin=os.path.join(workdir, "pointless.mtz"),
batch_info=batch_info,
out=out,
nproc=params.nproc,
nproc_each=params.batch.nproc_each,
batchjobs=None) # FIXME batchjobs
unused_files, reasons = cycles.run_cycles(xds_files)
used_files = set(xds_files).difference(set(unused_files))
print >> out
print >> out, " SUMMARY "
print >> out, "========================"
for i, files in enumerate((used_files, unused_files)):
print >> out, "\n%6s %4d files:\n" % (
("Used", "Unused")[i], len(files))
if len(files) == 0:
continue
maxlen_f = max(
map(lambda f: len(os.path.relpath(f, params.workdir)), files))
for f in files:
cell = cells[f]
merge_log = os.path.join(os.path.dirname(f),
"merging_stats.log")
try:
lines = open(merge_log).readlines()
resn = float(
filter(lambda x: x.startswith("Resolution:"),
lines)[0].split()[-1])
cmpl = float(
filter(lambda x: x.startswith("Completeness:"),
lines)[0].split()[-1].replace("%", ""))
except:
resn = float("nan")
cmpl = float("nan")
if i == 1: # print reason
print >> out, "%-15s" % reasons.get(f, "unknown"),
print >> out, ("%-" + str(maxlen_f) + "s") % os.path.relpath(
f, params.workdir), cell,
print >> out, "ISa=%5.1f" % correctlp.get_ISa(
os.path.join(os.path.dirname(f), "CORRECT.LP")),
print >> out, "Cmpl=%3.0f%%, Resn= %.1f" % (cmpl, resn)
aimless_log = os.path.join(cycles.current_working_dir(), "aimless.log")
print >> out, "\nFinal statistics:\n"
print >> out, aimless.snip_summary(aimless_log)
# Write summary
table = aimless.read_summary(aimless_log)
tkvals = lambda x: (x[0], x[1], x[2]) # overall, inner, outer
return [
[
cycles.get_last_cycle_number(),
cycles.current_working_dir(),
len(used_files),
dict(cmpl=tkvals(table["cmpl"]),
redundancy=tkvals(table["redundancy"]),
i_over_sigma=tkvals(table["i_over_sigma"]),
r_meas=tkvals(table["r_meas"]),
cc_half=tkvals(table["cc_half"]),
sig_ano=(float("nan"), ) * 3,
cc_ano=tkvals(table["cc_ano"]))
],
]
#print >>out, "\nRunning aimless"
#aimless.run_aimless(mtzin="pointless.mtz",
# wdir=workdir,
# anomalous=params.anomalous, d_min=params.d_min, prefix=None)
else:
print >> out, "Unknown program:", params.program
return []
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1, spots))
data_range = map(int, xdsinp_dict["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (iqr, q14,
cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print "DEBUG:: keep_frames=", keep_frames
print "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(x),
cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"), frame_selection=set(keep_frames))
# Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >> decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(
root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >> decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >> decilog, " Scaling failed."
return
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp,
check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(
root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(), symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
need_rescale = True
else:
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL",
"CORRECT_fullres.LP", "merging_stats.pkl",
"merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >> decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)):
os.remove(os.path.join(root, f))
revert_files(backup_needed,
bk_prefix,
wdir=root,
quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >> decilog, "xds_sequence finished at %s\n" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
init_lp = os.path.join(root, "INIT.LP")
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
if params.cell_prior.force: assert params.cell_prior.check
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
if params.cell_prior.sgnum > 0:
xs_prior = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
else:
xs_prior = None
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
try:
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if not kamo_test_installation.tst_xds():
print >> decilog, "XDS is not installed or expired!!"
return
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print >> decilog, " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print >> decilog, "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print >> decilog, " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS",
str(params.nproc)),
])
if params.mode == "initial":
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT")])
run_xds(wdir=root, show_progress=params.show_progress)
initlp = InitLp(init_lp)
first_bad = initlp.check_bad_first_frames()
if first_bad:
print >> decilog, " first frames look bad (too weak) exposure:", first_bad
new_data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
new_data_range[0] = first_bad[-1] + 1
print >> decilog, " changing DATA_RANGE= to", new_data_range
modify_xdsinp(xdsinp,
inp_params=[("JOB", "INIT"),
("DATA_RANGE",
"%d %d" % tuple(new_data_range))])
for f in xds_files.generated_by_INIT:
util.rotate_file(os.path.join(root, f), copy=False)
run_xds(wdir=root, show_progress=params.show_progress)
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1,
spots))
data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(
h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print >> decilog, "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (
iqr, q14, cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print >> decilog, "DEBUG:: keep_frames=", keep_frames
print >> decilog, "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(
lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(
x), cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"),
frame_selection=set(keep_frames))
# Indexing
if params.cell_prior.method == "use_first":
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
else:
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, "" # TODO indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.sgnum > 0:
# Check anyway
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(
xs_prior, xsxds, params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
if params.cell_prior.check:
print >> decilog, " Incompatible cell. Indexing failed."
return
else:
print >> decilog, " Warning: Incompatible cell."
elif params.cell_prior.method == "symm_constraint_only":
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Trying symmetry-constrained cell parameter:", cell
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
for f in xds_files.generated_by_IDXREF:
util.rotate_file(os.path.join(root, f),
copy=(f == "SPOT.XDS"))
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
# Check again
xsxds = XPARM(xparm).crystal_symmetry()
if not xsxds.unit_cell().is_similar_to(
xs_prior.unit_cell(), params.cell_prior.tol_length,
params.cell_prior.tol_angle):
print >> decilog, " Resulted in different cell. Indexing failed."
return
elif params.mode == "recycle":
print >> decilog, " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER",
"%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
pointless_integrate = {}
if params.use_pointless:
worker = Pointless()
pointless_integrate = worker.run_for_symm(
xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in pointless_integrate:
symm = pointless_integrate["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(symm.space_group(),
xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
else:
print >> decilog, " pointless failed."
flag_do_not_change_symm = False
if xs_prior and params.cell_prior.force:
modify_xdsinp(xdsinp,
inp_params=[("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum)])
flag_do_not_change_symm = True
elif params.cell_prior.method == "correct_only":
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(xs_prior, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is not None:
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Using given symmetry in CORRECT with symmetry constraints:", cell
modify_xdsinp(xdsinp,
inp_params=[
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
flag_do_not_change_symm = True
else:
print >> decilog, " Tried to use given symmetry in CORRECT, but cell in integration is incompatible."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xac_hkl):
print >> decilog, " CORRECT failed."
return
if not os.path.isfile(gxparm):
print >> decilog, " Refinement in CORRECT failed."
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(xac_hkl, os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
pointless_correct = worker.run_for_symm(
xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
pointless_best_symm = None
if "symm" in pointless_correct:
symm = pointless_correct["symm"]
need_rescale = False
if pointless_integrate.get("symm"):
symm_by_integrate = pointless_integrate["symm"]
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(),
symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
prob_integrate = pointless_integrate.get(
"laue_prob", float("nan"))
prob_correct = pointless_correct.get(
"laue_prob", float("nan"))
print >> decilog, " Prob(%s |INTEGRATE), Prob(%s |CORRECT) = %.4f, %.4f." % (
symm_by_integrate.space_group_info(),
symm.space_group_info(), prob_integrate,
prob_correct)
if prob_correct > prob_integrate:
need_rescale = True
pointless_best_symm = symm
else:
pointless_best_symm = symm_by_integrate
else:
need_rescale = True
pointless_best_symm = symm
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
if need_rescale and not flag_do_not_change_symm:
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)
])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(
xac_hkl,
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root,
show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
else:
print >> decilog, "ISa got worse= %.2f" % ISa
if pointless_best_symm:
xac_symm = XDS_ASCII(xac_hkl, read_data=False).symm
if not xtal.is_same_space_group_ignoring_enantiomorph(
xac_symm.space_group(),
pointless_best_symm.space_group()):
if xtal.is_same_laue_symmetry(
xac_symm.space_group(),
pointless_best_symm.space_group()):
tmp = "same Laue symmetry"
else:
tmp = "different Laue symmetry"
print >> decilog, "WARNING: symmetry in scaling is different from Pointless result (%s)." % tmp
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
except:
print >> decilog, traceback.format_exc()
finally:
print >> decilog, "\nxds_sequence finished at %s" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def merge_datasets(params, workdir, xds_files, cells, batchjobs):
if not os.path.exists(workdir): os.makedirs(workdir)
out = open(os.path.join(workdir, "merge.log"), "w")
if params.program == "xscale":
cycles = multi_merging.xscale.XscaleCycles(workdir,
anomalous_flag=params.anomalous,
d_min=params.d_min, d_max=params.d_max,
reject_method=params.reject_method,
reject_params=params.rejection,
xscale_params=params.xscale,
reference_file=params.reference_file,
out=out, nproc=params.nproc,
nproc_each=params.batch.nproc_each,
batchjobs=batchjobs if "deltacchalf" in params.batch.par_run else None)
unused_files, reasons = cycles.run_cycles(xds_files)
used_files = set(xds_files).difference(set(unused_files))
print >>out
print >>out, " SUMMARY "
print >>out, "========================"
for i, files in enumerate((used_files, unused_files)):
print >>out, "\n%6s %4d files:\n" % (("Used", "Unused")[i], len(files))
if len(files) == 0:
continue
maxlen_f = max(map(lambda f: len(os.path.relpath(f, params.workdir)), files))
for f in files:
cell = cells[f]
merge_log = os.path.join(os.path.dirname(f), "merging_stats.log")
try:
lines = open(merge_log).readlines()
resn = float(filter(lambda x:x.startswith("Resolution:"), lines)[0].split()[-1])
cmpl = float(filter(lambda x:x.startswith("Completeness:"), lines)[0].split()[-1].replace("%",""))
except:
resn = float("nan")
cmpl = float("nan")
if i == 1: # print reason
print >>out, "%-15s"%reasons.get(f, "unknown"),
print >>out, ("%-"+str(maxlen_f)+"s")%os.path.relpath(f, params.workdir), cell,
#print >>out, "ISa=%5.1f" % correctlp.get_ISa(os.path.join(os.path.dirname(f), "CORRECT.LP")),
print >>out, "Cmpl=%3.0f%%, Resn= %.1f" % (cmpl, resn)
ret = []
tkvals = lambda x: (x[-1], x[0], x[-2]) # overall, inner, outer
for i in xrange(1, cycles.get_last_cycle_number()+1):
wd = os.path.join(workdir, "run_%.2d"%i)
xscale_lp = os.path.join(wd, "XSCALE.LP")
table = xscalelp.read_stats_table(xscale_lp)
num_files = len(xscalelp.get_read_data(xscale_lp))
xtriage_logfile = os.path.join(wd, "ccp4", "logfile.log")
ret.append([i, wd, num_files,
dict(cmpl=tkvals(table["cmpl"]),
redundancy=tkvals(table["redundancy"]),
i_over_sigma=tkvals(table["i_over_sigma"]),
r_meas=tkvals(table["r_meas"]),
cc_half=tkvals(table["cc_half"]),
sig_ano=tkvals(table["sig_ano"]),
cc_ano=tkvals(table["cc_ano"]),
drange=tkvals(table["d_range"]),
lp=xscale_lp,
xtriage_log=xtriage.XtriageLogfile(xtriage_logfile))
])
xscale_lp = os.path.join(cycles.current_working_dir(), "XSCALE.LP")
print >>out, "\nFinal statistics:\n"
print >>out, xscalelp.snip_stats_table(xscale_lp)
return ret
elif params.program == "aimless":
worker = Pointless()
print >>out, "\nRunning pointless"
runinfo = worker.run_copy(hklout="pointless.mtz", wdir=workdir,
xdsin=xds_files,
logout=os.path.join(workdir, "pointless.log"),
tolerance=30)
# Table of file name -> Batch range
assert len(xds_files) == len(runinfo)
batch_info = collections.OrderedDict(map(lambda x: (x[0], (x[1][1:3])), zip(xds_files, runinfo)))
cycles = multi_merging.aimless.AimlessCycles(workdir,
anomalous_flag=params.anomalous,
d_min=params.d_min, d_max=params.d_max,
reject_method=params.reject_method,
cc_cutoff=params.rejection.lpstats.pwcc.abs_cutoff,
delta_cchalf_bin=params.rejection.delta_cchalf.bin,
mtzin=os.path.join(workdir, "pointless.mtz"),
batch_info=batch_info,
out=out, nproc=params.nproc,
nproc_each=params.batch.nproc_each,
batchjobs=batchjobs if "deltacchalf" in params.batch.par_run else None)
unused_files, reasons = cycles.run_cycles(xds_files)
used_files = set(xds_files).difference(set(unused_files))
print >>out
print >>out, " SUMMARY "
print >>out, "========================"
for i, files in enumerate((used_files, unused_files)):
print >>out, "\n%6s %4d files:\n" % (("Used", "Unused")[i], len(files))
if len(files) == 0:
continue
maxlen_f = max(map(lambda f: len(os.path.relpath(f, params.workdir)), files))
for f in files:
cell = cells[f]
merge_log = os.path.join(os.path.dirname(f), "merging_stats.log")
try:
lines = open(merge_log).readlines()
resn = float(filter(lambda x:x.startswith("Resolution:"), lines)[0].split()[-1])
cmpl = float(filter(lambda x:x.startswith("Completeness:"), lines)[0].split()[-1].replace("%",""))
except:
resn = float("nan")
cmpl = float("nan")
if i == 1: # print reason
print >>out, "%-15s"%reasons.get(f, "unknown"),
print >>out, ("%-"+str(maxlen_f)+"s")%os.path.relpath(f, params.workdir), cell,
print >>out, "ISa=%5.1f" % correctlp.get_ISa(os.path.join(os.path.dirname(f), "CORRECT.LP")),
print >>out, "Cmpl=%3.0f%%, Resn= %.1f" % (cmpl, resn)
aimless_log = os.path.join(cycles.current_working_dir(), "aimless.log")
print >>out, "\nFinal statistics:\n"
print >>out, aimless.snip_summary(aimless_log)
# Write summary
table = aimless.read_summary(aimless_log)
tkvals = lambda x: (x[0], x[1], x[2]) # overall, inner, outer
return [[cycles.get_last_cycle_number(), cycles.current_working_dir(), len(used_files),
dict(cmpl=tkvals(table["cmpl"]),
redundancy=tkvals(table["redundancy"]),
i_over_sigma=tkvals(table["i_over_sigma"]),
r_meas=tkvals(table["r_meas"]),
cc_half=tkvals(table["cc_half"]),
sig_ano=(float("nan"),)*3,
cc_ano=tkvals(table["cc_ano"]))], ]
#print >>out, "\nRunning aimless"
#aimless.run_aimless(mtzin="pointless.mtz",
# wdir=workdir,
# anomalous=params.anomalous, d_min=params.d_min, prefix=None)
else:
print >>out, "Unknown program:", params.program
return []
def run(params, out=sys.stdout):
cm = CheckMulti(topdir=params.topdir, xdsdirs=params.xdsdir, out=out)
cm.get_symms_from_xds_results()
cm.construct_graph(params.tol_length, params.tol_angle)
cm.group_xds_results()
print
ret = cm.grouped_dirs
if len(ret) == 0:
return cm
print >> out
print >> out, "About the largest group:"
for idx, wd in enumerate(ret[0]):
gxparm_xds = os.path.join(wd, "GXPARM.XDS")
correct_lp = os.path.join(wd, "CORRECT.LP")
print >> out, "%.3d %s" % (idx, os.path.relpath(wd, params.topdir)
if params.topdir is not None else wd),
if not os.path.isfile(gxparm_xds):
print >> out, "Unsuccessful"
continue
sg = sgtbx.space_group_info(XPARM(gxparm_xds).spacegroup)
clp = correctlp.CorrectLp(correct_lp)
if "all" in clp.table:
cmpl = clp.table["all"]["cmpl"][-1]
else:
cmpl = float("nan")
ISa = clp.a_b_ISa[-1]
print >> out, "%10s ISa=%5.2f Cmpl=%5.1f " % (sg, ISa, cmpl)
if params.do_pointless:
worker = Pointless()
files = map(lambda x: os.path.join(x, "INTEGRATE.HKL"), ret[0])
#print files
files = filter(lambda x: os.path.isfile(x), files)
print >> out, "\nRunning pointless for the largest member."
result = worker.run_for_symm(xdsin=files,
logout="pointless.log",
tolerance=10,
d_min=5)
if "symm" in result:
print >> out, " pointless suggested", result[
"symm"].space_group_info()
if 0:
import pylab
pos = nx.spring_layout(G)
#pos = nx.spectral_layout(G)
#pos = nx.circular_layout(G)
#nx.draw_networkx_nodes(G, pos, node_size = 100, nodelist=others, node_color = 'w')
nx.draw_networkx_nodes(G, pos, node_size=100, node_color='w')
nx.draw_networkx_edges(G, pos, width=1)
nx.draw_networkx_labels(G,
pos,
font_size=12,
font_family='sans-serif',
font_color='r')
pylab.xticks([])
pylab.yticks([])
pylab.savefig("network.png")
pylab.show()
return cm
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
decilog = multi_out()
decilog.register("log", open(os.path.join(root, "decision.log"), "a"), atexit_send_to=None)
print >>decilog, "xds_sequence started at %s in %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode=="initial" and params.resume and os.path.isfile(correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp, inp_params=[("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# To Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root, params.show_progress, decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >>decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell, params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length, params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >>decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS", "plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm+".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp, os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >>decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP",), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x:"%.3f"%x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d"%xparm_obj.spacegroup),])
print >>decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP",):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff+"_noscale")
revert_files(("XDS.INP",), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >>decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info()
sgnum = symm.space_group_info().type().number()
cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >>decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >>decilog, " Scaling failed."
return
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
print >>decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
print >>decilog, " (Original files are saved as *_fullres.*)"
else:
print >>decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(symm_by_integrate.space_group(), symm.space_group()):
print >>decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (symm.space_group_info(), symm_by_integrate.space_group_info())
need_rescale = True
else:
print >>decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info()
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL","CORRECT_fullres.LP",
"merging_stats.pkl","merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
print >>decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
print >>decilog, " (Original files are saved as *_fullres.*)"
else:
print >>decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP", "XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >>decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa!=last_ISa: # if improved or last_ISa is nan
print >>decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >>decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)): os.remove(os.path.join(root, f))
revert_files(backup_needed, bk_prefix, wdir=root, quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >>decilog, "xds_sequence finished at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
decilog.close()
