def get_colspot_result(frame_ranges, wdir):
assert all(map(lambda x: len(x) == 2 and x[0] <= x[1], frame_ranges))
# Check all needed files exist
if not check_needed_files(xds_files.needed_by_COLSPOT, wdir):
return
xdsinp = os.path.join(wdir, "XDS.INP")
backup_needed = xds_files.generated_by_COLSPOT + ("XDS.INP",)
# 1. Backup XDS.INP, etc. (Make copies; not renaming)
bk_prefix = make_backup(backup_needed, wdir=wdir)
try:
# 2. Modify XDS.INP
spot_ranges = map(lambda x: ("SPOT_RANGE", "%d %d" % tuple(x)), frame_ranges)
data_range = "%d %d" % (min(map(lambda x: min(x), frame_ranges)), max(map(lambda x: max(x), frame_ranges)))
modify_xdsinp(xdsinp, [("JOB", "COLSPOT"), ("DATA_RANGE", data_range)] + spot_ranges)
# 3. Run xds
p = subprocess.Popen("xds", cwd=wdir)
p.wait()
spotxds = SpotXds(os.path.join(wdir, "SPOT.XDS"))
finally:
# 6. Revert XDS.INP, etc.
revert_files(backup_needed, bk_prefix, wdir=wdir)
return spotxds
def run_xds_sequence(root, params):
tmpdir = None
if params.use_tmpdir_if_available:
tmpdir = util.get_temp_local_dir("xdskamo", min_gb=2) # TODO guess required tempdir size
if tmpdir is None:
print "Can't get temp dir with sufficient size."
# If tmpdir is not used
if tmpdir is None:
return xds_sequence(root, params)
print "Using %s as temp dir.." % tmpdir
# If tempdir is used
for f in glob.glob(os.path.join(root, "*")): shutil.copy2(f, tmpdir)
xdsinp = os.path.join(tmpdir, "XDS.INP")
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
# Make a link to data dir
org_data_template = xdsinp_dict["NAME_TEMPLATE_OF_DATA_FRAMES"]
ord_data_dir = os.path.dirname(org_data_template)
if not os.path.isabs(ord_data_dir): ord_data_dir = os.path.join(root, ord_data_dir)
datadir_lns = os.path.join(tmpdir, "data_loc")
os.symlink(ord_data_dir, datadir_lns)
# Modify XDS.INP
modify_xdsinp(xdsinp, inp_params=[("NAME_TEMPLATE_OF_DATA_FRAMES",
os.path.join("data_loc", os.path.basename(org_data_template)))])
try:
ret = xds_sequence(tmpdir, params)
finally:
# Revert XDS.INP
modify_xdsinp(xdsinp, inp_params=[("NAME_TEMPLATE_OF_DATA_FRAMES", org_data_template)])
# Remove link
os.remove(datadir_lns)
# Move to original directory
for f in glob.glob(os.path.join(tmpdir, "*")):
f_dest = os.path.join(root, os.path.relpath(f, tmpdir))
if os.path.isfile(f_dest):
# Copy only if newer
if os.stat(f).st_mtime > os.stat(f_dest).st_mtime:
shutil.copy2(f, root)
else:
print "%s already exists and not modified. skip." % f
os.remove(f)
else:
shutil.move(f, root)
# Remove tmpdir
shutil.rmtree(tmpdir)
return ret
def work(rootdir, xdsinp, orgxy):
workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
inpdir = os.path.normpath(os.path.dirname(xdsinp))
print workdir
os.makedirs(workdir)
shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"), os.path.join(workdir, "SPOT.XDS"))
shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
modify_xdsinp(os.path.join(workdir, "XDS.INP"), inp_params=[("JOB", "IDXREF"),
("ORGX", orgxy[0]),
("ORGY", orgxy[1]),
])
call("xds", wdir=workdir)
def work(rootdir, xdsinp, orgxy):
workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
inpdir = os.path.normpath(os.path.dirname(xdsinp))
print workdir
os.makedirs(workdir)
shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"),
os.path.join(workdir, "SPOT.XDS"))
shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
modify_xdsinp(os.path.join(workdir, "XDS.INP"),
inp_params=[
("JOB", "IDXREF"),
("ORGX", orgxy[0]),
("ORGY", orgxy[1]),
])
call("xds", wdir=workdir)
def cut_resolution(self, cycle_number):
def est_resol(xscale_hkl, res_params, plt_out):
iobs = XDS_ASCII(xscale_hkl, i_only=True).i_obs()
est = estimate_resolution_based_on_cc_half(iobs, res_params.cc_one_half_min,
res_params.cc_half_tol,
res_params.n_bins, log_out=self.out)
est.show_plot(False, plt_out)
if None not in (est.d_min, est.cc_at_d_min):
self.out.write("Best resolution cutoff= %.2f A @CC1/2= %.4f\n" % (est.d_min, est.cc_at_d_min))
else:
self.out.write("Can't decide resolution cutoff. No reflections??\n")
return est.d_min
# est_resol()
print >>self.out, "**** Determining resolution cutoff in run_%.2d ****" % cycle_number
last_wd = os.path.join(self.workdir_org, "run_%.2d"%cycle_number)
xscale_hkl = os.path.abspath(os.path.join(last_wd, "xscale.hkl"))
tmpwd = os.path.join(self.workdir_org, "run_%.2d_tmp"%cycle_number)
os.mkdir(tmpwd)
for i, cc_cut in enumerate((self.res_params.cc_one_half_min*.7, self.res_params.cc_one_half_min)):
self.res_params.cc_one_half_min = cc_cut
d_min = est_resol(xscale_hkl, self.res_params,
os.path.join(tmpwd, "ccfit_%d.pdf"%(i+1)))
if d_min is not None and d_min > self.d_min + 0.001:
for f in "XSCALE.INP", "XSCALE.LP": util.rotate_file(os.path.join(tmpwd, f))
inp_new = os.path.join(tmpwd, "XSCALE.INP")
shutil.copyfile(os.path.join(last_wd, "XSCALE.INP"), inp_new)
modify_xdsinp(inp_new, [make_bin_str(d_min, self.d_max).split("= ")])
try:
xscale.run_xscale(inp_new, cbf_to_dat=True, aniso_analysis=True,
use_tmpdir_if_available=self.xscale_params.use_tmpdir_if_available)
except:
print >>self.out, traceback.format_exc()
xscale_hkl = os.path.abspath(os.path.join(tmpwd, "xscale.hkl"))
if not os.path.isfile(os.path.join(tmpwd, "XSCALE.INP")):
for f in "XSCALE.INP", "XSCALE.LP", "xscale.hkl", "pointless.log", "ccp4":
os.symlink(os.path.relpath(os.path.join(last_wd, f), tmpwd),
os.path.join(tmpwd, f))
if d_min is not None:
self.dmin_est_at_cycles[cycle_number] = d_min
os.rename(tmpwd, os.path.join(self.workdir_org, "run_%.2d_%.2fA"%(cycle_number, d_min)))
def run(params):
xdsinp = "XDS.INP"
kwds = dict(get_xdsinp_keyword(xdsinp))
orgx_org, orgy_org = map(float, (kwds["ORGX"], kwds["ORGY"]))
dx, dy = params.dx, params.dy
if params.unit == "mm":
assert "QX" in kwds
assert "QY" in kwds
dx /= float(kwds["QX"])
dy /= float(kwds["QY"])
backup_needed = files.generated_by_IDXREF + ("XDS.INP",)
bk_prefix = make_backup(backup_needed)
try:
results = []
for i in xrange(-params.nx, params.nx+1):
for j in xrange(-params.ny, params.ny+1):
work_name = "bs_x%+.2d_y%+.2d" % (i, j)
orgx = orgx_org + i * dx
orgy = orgy_org + j * dy
print "Trying", orgx, orgy
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("ORGX", orgx),
("ORGY", orgy),
])
call("xds")
make_backup(backup_needed, work_name+"_")
results.append([work_name, orgx, orgy])
for ret in results:
print ret,
analyze_result(ret[0]+"_IDXREF.LP")
finally:
revert_files(backup_needed, bk_prefix)
def try_indexing_hard(wdir,
show_progress,
decilog,
known_sgnum=None,
known_cell=None,
tol_length=None,
tol_angle=None):
idxref_lp = os.path.join(wdir, "IDXREF.LP")
xdsinp = os.path.join(wdir, "XDS.INP")
lp_org = idxreflp.IdxrefLp(idxref_lp)
if lp_org.is_cell_maybe_half():
backup_needed = ("XDS.INP", ) + xds_files.generated_by_IDXREF
print >> decilog, " !! Cell may be halved. Trying doubled cell."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
cell = lp_org.deduce_correct_cell_based_on_integerness()
cell = " ".join(map(lambda x: "%.2f" % x, cell.parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "1"),
("UNIT_CELL_CONSTANTS", cell)])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
print >> decilog, " .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
modify_xdsinp(xdsinp,
inp_params=[("JOB", "IDXREF"), ("SEPMIN", "4"),
("CLUSTER_RADIUS", "2")])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
print >> decilog, " .. not solved. Give up."
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
else:
print >> decilog, " Now OK."
remove_backups(backup_needed, bk_prefix, wdir=wdir)
modify_xdsinp(xdsinp, inp_params=[
("SPACE_GROUP_NUMBER", "0"),
])
# If Cell hint exists, try to use it..
if known_sgnum > 0:
flag_try_cell_hint = False
xparm = os.path.join(wdir, "XPARM.XDS")
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(known_cell, known_sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds, tol_length,
tol_angle)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >> decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f" % x,
known_cell))),
("SPACE_GROUP_NUMBER", "%d" % known_sgnum),
])
run_xds(wdir=wdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[
("SPACE_GROUP_NUMBER", "0"),
])
def run(params, out):
print >> out, "Frames:", params.frames
backup_needed = xds_files.generated_by_DEFPIX + (
"XDS.INP",
"BKGINIT.cbf",
)
bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)
ret = {} # {frame: [matches, spots, predicted]}
try:
# run DEFPIX to limit resolution.
modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
[("JOB", "DEFPIX"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f" % params.d_min)])
p = subprocess.Popen("xds", cwd=params.xdsdir)
p.wait()
# copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
os.path.join(params.xdsdir, "BKGINIT.cbf"))
for frame in params.frames:
print >> out, "Frame %d" % frame
print >> out, "====================\n"
# search spots
if params.spotfinder == "xds":
spotxds = get_colspot_result(frame_ranges=[
[frame, frame],
],
wdir=params.xdsdir)
spots = map(lambda x: x[:2],
spotxds.collected_spots(with_resolution=False))
else:
raise "Sorry!"
# run INTEGRATE to collect predicted coords
integrate_results = xds_predict_mitai.run(
param_source=os.path.join(params.xdsdir, "XPARM.XDS"),
frame_num=frame,
wdir=params.xdsdir,
need_adx=False,
sigmar=params.sigmar,
sigmab=params.sigmab)
# read predicted coords
tmp = filter(lambda x: x.endswith(".HKL"), integrate_results)
if len(tmp) == 0:
print >> out, "Integration failed!"
ret[frame] = (0, len(spots), 0)
continue
integrate_hkl = tmp[0]
cols = integrate_hkl_as_flex.reader(integrate_hkl, [],
False).get_column_names()
i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
predicted = []
for l in open(integrate_hkl):
if l.startswith("!"): continue
sp = l.split()
predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))
# compare them
nmatch = calc_matches(
spots, predicted, params.distance_limit_in_px,
open(
os.path.join(params.xdsdir,
"matched_predicted_%.4d.adx" % frame), "w"))
#nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
# open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))
ret[frame] = (nmatch, len(spots), len(predicted))
finally:
revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)
print >> out
for frame in sorted(ret):
nmatch, nspots, npredicted = ret[frame]
print >> out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (
frame, nmatch, npredicted, 100. * float(nmatch) /
npredicted if npredicted > 0 else float("nan"))
print >> out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (
frame, nmatch, nspots,
100. * float(nmatch) / nspots if nspots > 0 else float("nan"))
print >> out
return ret
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
init_lp = os.path.join(root, "INIT.LP")
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
if params.cell_prior.force: assert params.cell_prior.check
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
if params.cell_prior.sgnum > 0:
xs_prior = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
else:
xs_prior = None
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
try:
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if not kamo_test_installation.tst_xds():
print >> decilog, "XDS is not installed or expired!!"
return
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print >> decilog, " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print >> decilog, "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print >> decilog, " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS",
str(params.nproc)),
])
if params.mode == "initial":
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT")])
run_xds(wdir=root, show_progress=params.show_progress)
initlp = InitLp(init_lp)
first_bad = initlp.check_bad_first_frames()
if first_bad:
print >> decilog, " first frames look bad (too weak) exposure:", first_bad
new_data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
new_data_range[0] = first_bad[-1] + 1
print >> decilog, " changing DATA_RANGE= to", new_data_range
modify_xdsinp(xdsinp,
inp_params=[("JOB", "INIT"),
("DATA_RANGE",
"%d %d" % tuple(new_data_range))])
for f in xds_files.generated_by_INIT:
util.rotate_file(os.path.join(root, f), copy=False)
run_xds(wdir=root, show_progress=params.show_progress)
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1,
spots))
data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(
h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print >> decilog, "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (
iqr, q14, cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print >> decilog, "DEBUG:: keep_frames=", keep_frames
print >> decilog, "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(
lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(
x), cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"),
frame_selection=set(keep_frames))
# Indexing
if params.cell_prior.method == "use_first":
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
else:
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, "" # TODO indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.sgnum > 0:
# Check anyway
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(
xs_prior, xsxds, params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
if params.cell_prior.check:
print >> decilog, " Incompatible cell. Indexing failed."
return
else:
print >> decilog, " Warning: Incompatible cell."
elif params.cell_prior.method == "symm_constraint_only":
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Trying symmetry-constrained cell parameter:", cell
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
for f in xds_files.generated_by_IDXREF:
util.rotate_file(os.path.join(root, f),
copy=(f == "SPOT.XDS"))
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
# Check again
xsxds = XPARM(xparm).crystal_symmetry()
if not xsxds.unit_cell().is_similar_to(
xs_prior.unit_cell(), params.cell_prior.tol_length,
params.cell_prior.tol_angle):
print >> decilog, " Resulted in different cell. Indexing failed."
return
elif params.mode == "recycle":
print >> decilog, " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER",
"%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
pointless_integrate = {}
if params.use_pointless:
worker = Pointless()
pointless_integrate = worker.run_for_symm(
xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in pointless_integrate:
symm = pointless_integrate["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(symm.space_group(),
xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
else:
print >> decilog, " pointless failed."
flag_do_not_change_symm = False
if xs_prior and params.cell_prior.force:
modify_xdsinp(xdsinp,
inp_params=[("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum)])
flag_do_not_change_symm = True
elif params.cell_prior.method == "correct_only":
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(xs_prior, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is not None:
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Using given symmetry in CORRECT with symmetry constraints:", cell
modify_xdsinp(xdsinp,
inp_params=[
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
flag_do_not_change_symm = True
else:
print >> decilog, " Tried to use given symmetry in CORRECT, but cell in integration is incompatible."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xac_hkl):
print >> decilog, " CORRECT failed."
return
if not os.path.isfile(gxparm):
print >> decilog, " Refinement in CORRECT failed."
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(xac_hkl, os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
pointless_correct = worker.run_for_symm(
xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
pointless_best_symm = None
if "symm" in pointless_correct:
symm = pointless_correct["symm"]
need_rescale = False
if pointless_integrate.get("symm"):
symm_by_integrate = pointless_integrate["symm"]
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(),
symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
prob_integrate = pointless_integrate.get(
"laue_prob", float("nan"))
prob_correct = pointless_correct.get(
"laue_prob", float("nan"))
print >> decilog, " Prob(%s |INTEGRATE), Prob(%s |CORRECT) = %.4f, %.4f." % (
symm_by_integrate.space_group_info(),
symm.space_group_info(), prob_integrate,
prob_correct)
if prob_correct > prob_integrate:
need_rescale = True
pointless_best_symm = symm
else:
pointless_best_symm = symm_by_integrate
else:
need_rescale = True
pointless_best_symm = symm
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
if need_rescale and not flag_do_not_change_symm:
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)
])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(
xac_hkl,
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root,
show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
else:
print >> decilog, "ISa got worse= %.2f" % ISa
if pointless_best_symm:
xac_symm = XDS_ASCII(xac_hkl, read_data=False).symm
if not xtal.is_same_space_group_ignoring_enantiomorph(
xac_symm.space_group(),
pointless_best_symm.space_group()):
if xtal.is_same_laue_symmetry(
xac_symm.space_group(),
pointless_best_symm.space_group()):
tmp = "same Laue symmetry"
else:
tmp = "different Laue symmetry"
print >> decilog, "WARNING: symmetry in scaling is different from Pointless result (%s)." % tmp
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
except:
print >> decilog, traceback.format_exc()
finally:
print >> decilog, "\nxds_sequence finished at %s" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def find_spots(img_file, params):
"""
Use XDS to locate spots.
If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
"""
# Test if ramdisk available
if os.path.isdir("/dev/shm"):
work_dir = tempfile.mkdtemp(
prefix="shika_x_" +
os.path.splitext(os.path.basename(img_file))[0],
dir="/dev/shm")
else:
work_dir = os.path.join(
params.work_dir,
"xds_" + os.path.splitext(os.path.basename(img_file))[0])
xdsinp = os.path.join(work_dir, "XDS.INP")
spot_xds = os.path.join(work_dir, "SPOT.XDS")
if not os.path.exists(work_dir):
os.mkdir(work_dir)
template_str, min_frame, max_frame = dataset.group_img_files_template(
[img_file])[0]
im = XIO.Image(img_file)
# Remove lines if None (means to use default)
params_maps = [
("strong_pixel", "STRONG_PIXEL="),
("minimum_number_of_pixels_in_a_spot",
"MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
("background_pixel", "BACKGROUND_PIXEL="),
("maximum_number_of_strong_pixels",
"MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
("reflecting_range", "REFLECTING_RANGE="),
("value_range_for_trusted_detector_pixels",
"VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
]
tmp = xds_inp_template.splitlines()
for p, x in params_maps:
if getattr(params.xds, p) is None:
tmp = filter(lambda s: not s.startswith(x), tmp)
inp_template = "\n".join(tmp)
# Prepare XDS.INP
inp_str = inp_template % dict(
template=os.path.relpath(template_str, work_dir),
framenum=min_frame,
orgx=im.header["BeamX"] / im.header["PixelX"],
orgy=im.header["BeamY"] / im.header["PixelY"],
distance=im.header["Distance"],
osc_range=im.header["PhiWidth"],
wavelength=im.header["Wavelength"],
strong_pixel=params.xds.strong_pixel,
min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
background_pixel=params.xds.background_pixel,
max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
spot_maximum_centroid=params.xds.spot_maximum_centroid,
reflecting_range=params.xds.reflecting_range,
nx=im.header["Width"],
ny=im.header["Height"],
qx=im.header["PixelX"],
qy=im.header["PixelY"],
defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1])
open(xdsinp, "w").write(inp_str)
if params.xds.do_defpix:
xp = xparm.XPARM()
xp.set_info_from_xdsinp(xdsinp)
open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
modify_xdsinp(xdsinp,
inp_params=[("JOB", "XYCORR INIT DEFPIX"),
("INCLUDE_RESOLUTION_RANGE",
res_range_for_xds(params.distl.res.outer,
params.distl.res.inner))])
call("xds",
wdir=work_dir,
stdout=open(os.path.join(work_dir, "xds.log"), "w"))
shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"),
os.path.join(work_dir, "BKGINIT.cbf"))
modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
else:
modify_xdsinp(xdsinp,
inp_params=[
("TRUSTED_REGION",
res_range_to_trusted_region(params.distl.res.outer,
params.distl.res.inner,
im.header))
])
open(os.path.join(work_dir, "xds.log"), "w").write("")
# Run XDS
rotate_file(spot_xds)
call("xds",
wdir=work_dir,
stdout=open(os.path.join(work_dir, "xds.log"), "a"))
# Extract results
spots = [] # (x, y, d, intensity)
if os.path.isfile(spot_xds):
for l in open(spot_xds):
x, y, z, intensity = map(lambda x: float(x), l.strip().split())
d = coord_to_resol(x, y, im.header)
spots.append((x, y, d, intensity))
# Delete dir
shutil.rmtree(work_dir)
return spots
def find_spots(img_file, params):
"""
Use XDS to locate spots.
If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
"""
# Test if ramdisk available
if os.path.isdir("/dev/shm"):
work_dir = tempfile.mkdtemp(prefix="shika_x_"+os.path.splitext(os.path.basename(img_file))[0], dir="/dev/shm")
else:
work_dir = os.path.join(params.work_dir, "xds_"+os.path.splitext(os.path.basename(img_file))[0])
xdsinp = os.path.join(work_dir, "XDS.INP")
spot_xds = os.path.join(work_dir, "SPOT.XDS")
if not os.path.exists(work_dir):
os.mkdir(work_dir)
template_str, min_frame, max_frame = dataset.group_img_files_template([img_file])[0]
im = XIO.Image(img_file)
# Remove lines if None (means to use default)
params_maps = [("strong_pixel", "STRONG_PIXEL="),
("minimum_number_of_pixels_in_a_spot", "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
("background_pixel", "BACKGROUND_PIXEL="),
("maximum_number_of_strong_pixels", "MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
("reflecting_range", "REFLECTING_RANGE="),
("value_range_for_trusted_detector_pixels", "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
]
tmp = xds_inp_template.splitlines()
for p, x in params_maps:
if getattr(params.xds, p) is None:
tmp = filter(lambda s:not s.startswith(x), tmp)
inp_template = "\n".join(tmp)
# Prepare XDS.INP
inp_str = inp_template%dict(template=os.path.relpath(template_str, work_dir),
framenum=min_frame,
orgx=im.header["BeamX"]/im.header["PixelX"],
orgy=im.header["BeamY"]/im.header["PixelY"],
distance=im.header["Distance"],
osc_range=im.header["PhiWidth"],
wavelength=im.header["Wavelength"],
strong_pixel=params.xds.strong_pixel,
min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
background_pixel=params.xds.background_pixel,
max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
spot_maximum_centroid=params.xds.spot_maximum_centroid,
reflecting_range=params.xds.reflecting_range,
nx=im.header["Width"], ny=im.header["Height"],
qx=im.header["PixelX"], qy=im.header["PixelY"],
defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1]
)
open(xdsinp, "w").write(inp_str)
if params.xds.do_defpix:
xp = xparm.XPARM()
xp.set_info_from_xdsinp(xdsinp)
open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT DEFPIX"),
("INCLUDE_RESOLUTION_RANGE", res_range_for_xds(params.distl.res.outer, params.distl.res.inner))
])
call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "w"))
shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"), os.path.join(work_dir, "BKGINIT.cbf"))
modify_xdsinp(xdsinp, inp_params=[("JOB","COLSPOT")])
else:
modify_xdsinp(xdsinp, inp_params=[("TRUSTED_REGION", res_range_to_trusted_region(params.distl.res.outer, params.distl.res.inner, im.header))
])
open(os.path.join(work_dir, "xds.log"), "w").write("")
# Run XDS
rotate_file(spot_xds)
call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "a"))
# Extract results
spots = [] # (x, y, d, intensity)
if os.path.isfile(spot_xds):
for l in open(spot_xds):
x, y, z, intensity = map(lambda x:float(x), l.strip().split())
d = coord_to_resol(x, y, im.header)
spots.append((x, y, d, intensity))
# Delete dir
shutil.rmtree(work_dir)
return spots
def rescale_with_specified_symm(topdir,
dirs,
symms,
out,
sgnum=None,
reference_symm=None):
assert (sgnum, reference_symm).count(None) == 1
if sgnum is not None:
sgnum_laue = sgtbx.space_group_info(sgnum).group(
).build_derived_reflection_intensity_group(False).type().number()
matches = filter(
lambda x: x.reflection_intensity_symmetry(False).space_group_info(
).type().number() == sgnum_laue, symms)
matched_cells = numpy.array(
map(lambda x: x.unit_cell().parameters(), matches))
median_cell = map(lambda x: numpy.median(matched_cells[:, x]),
xrange(6))
reference_symm = crystal.symmetry(median_cell, sgnum)
else:
sgnum = reference_symm.space_group_info().type().number()
sgnum_laue = reference_symm.space_group(
).build_derived_reflection_intensity_group(False).type().number()
print >> out
print >> out, "Re-scaling with specified symmetry:", reference_symm.space_group_info(
).symbol_and_number()
print >> out, " reference cell:", reference_symm.unit_cell()
print >> out
print >> out
cells = {} # cell and file
for sym, wd in zip(symms, dirs):
print >> out, os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
continue
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (
os.path.basename(xac_file), xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(),
0.1, 10):
print >> out, " Already scaled with specified symmetry"
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()),
xac_file)
continue
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
20)
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = reference_symm.space_group().average_unit_cell(
xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT,
wdir=wd,
quiet=True)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f),
os.path.join(wd, f + "_rescale"))
revert_files(xds_files.generated_by_CORRECT,
bk_prefix,
wdir=wd,
quiet=True)
new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
print "OK:", cells[wd][0]
else:
print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
continue
return cells, reference_symm
def run(params, out):
print >>out, "Frames:", params.frames
backup_needed = xds_files.generated_by_DEFPIX + ("XDS.INP","BKGINIT.cbf",)
bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)
ret = {} # {frame: [matches, spots, predicted]}
try:
# run DEFPIX to limit resolution.
modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
[("JOB", "DEFPIX"), ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%params.d_min)])
p = subprocess.Popen("xds", cwd=params.xdsdir)
p.wait()
# copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
os.path.join(params.xdsdir, "BKGINIT.cbf"))
for frame in params.frames:
print >>out, "Frame %d" % frame
print >>out, "====================\n"
# search spots
if params.spotfinder == "xds":
spotxds = get_colspot_result(frame_ranges=[[frame, frame],], wdir=params.xdsdir)
spots = map(lambda x: x[:2], spotxds.collected_spots(with_resolution=False))
else:
raise "Sorry!"
# run INTEGRATE to collect predicted coords
integrate_results = xds_predict_mitai.run(param_source=os.path.join(params.xdsdir, "XPARM.XDS"),
frame_num=frame,
wdir=params.xdsdir, need_adx=False,
sigmar=params.sigmar, sigmab=params.sigmab)
# read predicted coords
tmp = filter(lambda x:x.endswith(".HKL"), integrate_results)
if len(tmp) == 0:
print >>out, "Integration failed!"
ret[frame] = (0, len(spots), 0)
continue
integrate_hkl = tmp[0]
cols = integrate_hkl_as_flex.reader(integrate_hkl, [], False).get_column_names()
i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
predicted = []
for l in open(integrate_hkl):
if l.startswith("!"): continue
sp = l.split()
predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))
# compare them
nmatch = calc_matches(spots, predicted, params.distance_limit_in_px,
open(os.path.join(params.xdsdir, "matched_predicted_%.4d.adx"%frame), "w"))
#nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
# open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))
ret[frame] = (nmatch, len(spots), len(predicted))
finally:
revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)
print >>out
for frame in sorted(ret):
nmatch, nspots, npredicted = ret[frame]
print >>out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (frame, nmatch, npredicted,
100.*float(nmatch)/npredicted if npredicted>0 else float("nan"))
print >>out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (frame, nmatch, nspots,
100.*float(nmatch)/nspots if nspots>0 else float("nan"))
print >>out
return ret
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1, spots))
data_range = map(int, xdsinp_dict["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (iqr, q14,
cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print "DEBUG:: keep_frames=", keep_frames
print "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(x),
cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"), frame_selection=set(keep_frames))
# Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >> decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(
root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >> decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >> decilog, " Scaling failed."
return
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp,
check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(
root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(), symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
need_rescale = True
else:
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL",
"CORRECT_fullres.LP", "merging_stats.pkl",
"merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >> decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)):
os.remove(os.path.join(root, f))
revert_files(backup_needed,
bk_prefix,
wdir=root,
quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >> decilog, "xds_sequence finished at %s\n" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def rescale_with_specified_symm(topdir, dirs, symms, out, sgnum=None, reference_symm=None):
assert (sgnum, reference_symm).count(None) == 1
if sgnum is not None:
sgnum_laue = sgtbx.space_group_info(sgnum).group().build_derived_reflection_intensity_group(False).type().number()
matches = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)
matched_cells = numpy.array(map(lambda x: x.unit_cell().parameters(), matches))
median_cell = map(lambda x: numpy.median(matched_cells[:,x]), xrange(6))
reference_symm = crystal.symmetry(median_cell, sgnum)
else:
sgnum = reference_symm.space_group_info().type().number()
sgnum_laue = reference_symm.space_group().build_derived_reflection_intensity_group(False).type().number()
print >>out
print >>out, "Re-scaling with specified symmetry:", reference_symm.space_group_info().symbol_and_number()
print >>out, " reference cell:", reference_symm.unit_cell()
print >>out
print >>out
cells = {} # cell and file
for sym, wd in zip(symms, dirs):
print >>out, os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >>out, "Can't find XDS_ASCII file in %s" % wd
continue
xac = XDS_ASCII(xac_file, read_data=False)
print >>out, "%s %s (%s)" % (os.path.basename(xac_file), xac.symm.space_group_info(),
",".join(map(lambda x: "%.2f"%x, xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1, 10):
print >>out, " Already scaled with specified symmetry"
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()), xac_file)
continue
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)
if len(cosets.combined_cb_ops())==0:
print >>out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = reference_symm.space_group().average_unit_cell(xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f"%x, newcell.parameters()))
print >>out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT, wdir=wd, quiet=True)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f), os.path.join(wd, f+"_rescale"))
revert_files(xds_files.generated_by_CORRECT, bk_prefix, wdir=wd, quiet=True)
new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
print "OK:", cells[wd][0]
else:
print >>out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
continue
return cells, reference_symm
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in+ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir): os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >>decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >>decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True, anomalous=params.anomalous,
spot_range=None, minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx, orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f"%params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f"%params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d"%params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f"%params.sepmin),
("CLUSTER_RADIUS", "%.2f"%params.cluster_radius),
("INDEX_ERROR", "%.4f"%params.index_error),
("INDEX_MAGNITUDE", "%d"%params.index_magnitude),
("INDEX_QUALITY", "%.2f"%params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
("VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels))
])
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >>decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f"%x, params.cell))),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >>decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min)),
])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell)))]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum)] + tmp)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f+".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(f, read_columns=["IOBS","SIGMA","XCAL","YCAL","RLP","PEAK","MAXC"])
pickle.dump(inhkl, open(f+".pkl", "w"), -1)
except ProcFailed, e:
print >>decilog, "Processing failed: %s" % e.message
def try_indexing_hard(wdir, show_progress, decilog,
known_sgnum=None, known_cell=None, tol_length=None, tol_angle=None):
idxref_lp = os.path.join(wdir, "IDXREF.LP")
xdsinp = os.path.join(wdir, "XDS.INP")
lp_org = idxreflp.IdxrefLp(idxref_lp)
if lp_org.is_cell_maybe_half():
backup_needed = ("XDS.INP",) + xds_files.generated_by_IDXREF
print >>decilog, " !! Cell may be halved. Trying doubled cell."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
cell = lp_org.deduce_correct_cell_based_on_integerness()
cell = " ".join(map(lambda x:"%.2f"%x, cell.parameters()))
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "1"),
("UNIT_CELL_CONSTANTS", cell)
])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
print >>decilog, " .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SEPMIN", "4"),
("CLUSTER_RADIUS", "2")
])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
print >>decilog, " .. not solved. Give up."
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
else:
print >>decilog, " Now OK."
remove_backups(backup_needed, bk_prefix, wdir=wdir)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
# If Cell hint exists, try to use it..
if known_sgnum > 0:
flag_try_cell_hint = False
xparm = os.path.join(wdir, "XPARM.XDS")
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(known_cell, known_sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
tol_length, tol_angle)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >>decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f"%x, known_cell))),
("SPACE_GROUP_NUMBER", "%d"%known_sgnum),
])
run_xds(wdir=wdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
decilog = multi_out()
decilog.register("log", open(os.path.join(root, "decision.log"), "a"), atexit_send_to=None)
print >>decilog, "xds_sequence started at %s in %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode=="initial" and params.resume and os.path.isfile(correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp, inp_params=[("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# To Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root, params.show_progress, decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >>decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell, params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length, params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >>decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS", "plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm+".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp, os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >>decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP",), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x:"%.3f"%x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d"%xparm_obj.spacegroup),])
print >>decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP",):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff+"_noscale")
revert_files(("XDS.INP",), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >>decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info()
sgnum = symm.space_group_info().type().number()
cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >>decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >>decilog, " Scaling failed."
return
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
print >>decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
print >>decilog, " (Original files are saved as *_fullres.*)"
else:
print >>decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(symm_by_integrate.space_group(), symm.space_group()):
print >>decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (symm.space_group_info(), symm_by_integrate.space_group_info())
need_rescale = True
else:
print >>decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info()
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL","CORRECT_fullres.LP",
"merging_stats.pkl","merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
print >>decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
print >>decilog, " (Original files are saved as *_fullres.*)"
else:
print >>decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP", "XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >>decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa!=last_ISa: # if improved or last_ISa is nan
print >>decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >>decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)): os.remove(os.path.join(root, f))
revert_files(backup_needed, bk_prefix, wdir=root, quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >>decilog, "xds_sequence finished at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
decilog.close()
def rescale_with_specified_symm_worker(sym_wd_wdr,
topdir,
log_out,
reference_symm,
sgnum,
sgnum_laue,
prep_dials_files=False):
# XXX Unsafe if multiple processes run this function for the same target directory at the same time
sym, wd, wdr = sym_wd_wdr
out = StringIO()
print >> out, os.path.relpath(wd, topdir),
# Find appropriate data # XXX not works for DIALS data!!
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
log_out.write(out.getvalue())
log_out.flush()
return (wd, None)
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1,
10):
print >> out, " Already scaled with specified symmetry"
log_out.write(out.getvalue())
log_out.flush()
if wd != wdr: shutil.copy2(xac_file, wdr)
if prep_dials_files: prepare_dials_files(wd, out, moveto=wdr)
return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
os.path.join(wdr, os.path.basename(xac_file))))
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
log_out.write(out.getvalue())
log_out.flush()
return (wdr, None)
newcell = reference_symm.space_group().average_unit_cell(
xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT,
wdir=wd,
quiet=True)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f),
os.path.join(wdr, f + "_rescale"))
revert_files(xds_files.generated_by_CORRECT,
bk_prefix,
wdir=wd,
quiet=True)
new_xac = os.path.join(wdr, "XDS_ASCII.HKL_rescale")
if prep_dials_files:
prepare_dials_files(wd,
out,
space_group=reference_symm.space_group(),
reindex_op=cosets.combined_cb_ops()[0],
moveto=wdr)
ret = None
if os.path.isfile(new_xac):
ret = (XDS_ASCII(new_xac,
read_data=False).symm.unit_cell().parameters(),
new_xac)
print >> out, " OK:", ret[0]
else:
print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
return (wd, ret)
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in + ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >> decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >> decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(
img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True,
anomalous=params.anomalous,
spot_range=None,
minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx,
orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance,
)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f" % params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f" % params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d" % params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f" % params.sepmin),
("CLUSTER_RADIUS", "%.2f" % params.cluster_radius),
("INDEX_ERROR", "%.4f" % params.index_error),
("INDEX_MAGNITUDE", "%d" % params.index_magnitude),
("INDEX_QUALITY", "%.2f" % params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
(
"VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS",
"%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels),
),
],
)
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
flag_try_cell_hint = True
if flag_try_cell_hint:
print >> decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
],
)
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0")])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >> decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
],
)
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min))])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE")])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [
("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
]
+ tmp,
)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >> decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle:
x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f + ".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >> decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(
f, read_columns=["IOBS", "SIGMA", "XCAL", "YCAL", "RLP", "PEAK", "MAXC"]
)
pickle.dump(inhkl, open(f + ".pkl", "w"), -1)
except ProcFailed, e:
print >> decilog, "Processing failed: %s" % e.message
