def get_symms_from_xds_results(self):
if self.xdsdirs is not None:
xdsdirs = filter(lambda x: os.path.isfile(os.path.join(x, "GXPARM.XDS")), self.xdsdirs)
else:
xdsdirs = map(lambda x: x[0], filter(lambda x: "GXPARM.XDS" in x[2], os.walk(self.topdir)))
symms = []
print >>self.out, "Idx Dir Cell P1Cell"
idx = 0
for root in xdsdirs:
print >>self.out, "%.3d"%idx,
print >>self.out, os.path.relpath(root, self.topdir) if self.topdir is not None else root,
gxparm_xds = os.path.join(root, "GXPARM.XDS")
p1cell = correctlp.get_P1_cell(os.path.join(root, "CORRECT.LP"))
try:
xparm = XPARM(gxparm_xds)
except ValueError:
print >>self.out, "Invalid xparm format:", gxparm_xds
continue
xs = xparm.crystal_symmetry()
self.dirs.append(root)
self.p1cells.append(p1cell)
self.symms.append(xs)
print >>self.out, xs.space_group_info(), xs.unit_cell(), p1cell
idx += 1
assert len(self.dirs) == len(self.symms) == len(self.p1cells)
def get_symms_from_xds_results(self):
if self.xdsdirs is not None:
xdsdirs = filter(
lambda x: os.path.isfile(os.path.join(x, "GXPARM.XDS")),
self.xdsdirs)
else:
xdsdirs = map(
lambda x: x[0],
filter(lambda x: "GXPARM.XDS" in x[2], os.walk(self.topdir)))
symms = []
print >> self.out, "Idx Dir Cell P1Cell"
idx = 0
for root in xdsdirs:
print >> self.out, "%.3d" % idx,
print >> self.out, os.path.relpath(
root, self.topdir) if self.topdir is not None else root,
gxparm_xds = os.path.join(root, "GXPARM.XDS")
p1cell = correctlp.get_P1_cell(os.path.join(root, "CORRECT.LP"),
force_obtuse_angle=True)
try:
xparm = XPARM(gxparm_xds)
except ValueError:
print >> self.out, "Invalid xparm format:", gxparm_xds
continue
xs = xparm.crystal_symmetry()
self.dirs.append(root)
self.p1cells.append(p1cell)
self.symms.append(xs)
print >> self.out, xs.space_group_info(), xs.unit_cell(), p1cell
idx += 1
assert len(self.dirs) == len(self.symms) == len(self.p1cells)
def run(bkgpix_in, xparm_in, nbins):
bkgpix, nx, ny = cbf.load_cbf_as_flex(bkgpix_in)
xparm = XPARM(xparm_in)
d_min = calc_edge_resolution(xparm, nx, ny)
print "# edge resolution=", d_min
s2_step = (1./d_min**2) / nbins
bins = [[] for i in xrange(nbins)]
for i in xrange(0, bkgpix.size(), 2):
val = bkgpix[i]
if val < 0:
continue
x,y = i%nx, int(i/nx)
d = xy_to_d(x, y, xparm)
if d < d_min:
continue
s2 = 1./d**2
idx = int(s2/s2_step)
if idx >= nbins: idx = nbins - 1
bins[idx].append(val/100.)
for i in xrange(nbins):
dmax, dmin = 1./numpy.sqrt(i*s2_step), 1./numpy.sqrt((i+1)*s2_step)
print "%7.2f %7.2f %.4f" % (dmax, dmin, sum(bins[i])/len(bins[i]))
def run(xds_inp):
xp = XPARM()
xp.set_info_from_xdsinp_or_inpstr(xdsinp=xds_inp)
xp.update_cell_based_on_axes()
print xp.xparm_str().rstrip()
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1, spots))
data_range = map(int, xdsinp_dict["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (iqr, q14,
cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print "DEBUG:: keep_frames=", keep_frames
print "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(x),
cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"), frame_selection=set(keep_frames))
# Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >> decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(
root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >> decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >> decilog, " Scaling failed."
return
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp,
check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(
root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(), symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
need_rescale = True
else:
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL",
"CORRECT_fullres.LP", "merging_stats.pkl",
"merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >> decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)):
os.remove(os.path.join(root, f))
revert_files(backup_needed,
bk_prefix,
wdir=root,
quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >> decilog, "xds_sequence finished at %s\n" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def try_indexing_hard(wdir,
show_progress,
decilog,
known_sgnum=None,
known_cell=None,
tol_length=None,
tol_angle=None):
idxref_lp = os.path.join(wdir, "IDXREF.LP")
xdsinp = os.path.join(wdir, "XDS.INP")
lp_org = idxreflp.IdxrefLp(idxref_lp)
if lp_org.is_cell_maybe_half():
backup_needed = ("XDS.INP", ) + xds_files.generated_by_IDXREF
print >> decilog, " !! Cell may be halved. Trying doubled cell."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
cell = lp_org.deduce_correct_cell_based_on_integerness()
cell = " ".join(map(lambda x: "%.2f" % x, cell.parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "1"),
("UNIT_CELL_CONSTANTS", cell)])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
print >> decilog, " .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
modify_xdsinp(xdsinp,
inp_params=[("JOB", "IDXREF"), ("SEPMIN", "4"),
("CLUSTER_RADIUS", "2")])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
print >> decilog, " .. not solved. Give up."
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
else:
print >> decilog, " Now OK."
remove_backups(backup_needed, bk_prefix, wdir=wdir)
modify_xdsinp(xdsinp, inp_params=[
("SPACE_GROUP_NUMBER", "0"),
])
# If Cell hint exists, try to use it..
if known_sgnum > 0:
flag_try_cell_hint = False
xparm = os.path.join(wdir, "XPARM.XDS")
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(known_cell, known_sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds, tol_length,
tol_angle)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >> decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f" % x,
known_cell))),
("SPACE_GROUP_NUMBER", "%d" % known_sgnum),
])
run_xds(wdir=wdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[
("SPACE_GROUP_NUMBER", "0"),
])
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
init_lp = os.path.join(root, "INIT.LP")
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
if params.cell_prior.force: assert params.cell_prior.check
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
if params.cell_prior.sgnum > 0:
xs_prior = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
else:
xs_prior = None
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
try:
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if not kamo_test_installation.tst_xds():
print >> decilog, "XDS is not installed or expired!!"
return
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print >> decilog, " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print >> decilog, "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print >> decilog, " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS",
str(params.nproc)),
])
if params.mode == "initial":
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT")])
run_xds(wdir=root, show_progress=params.show_progress)
initlp = InitLp(init_lp)
first_bad = initlp.check_bad_first_frames()
if first_bad:
print >> decilog, " first frames look bad (too weak) exposure:", first_bad
new_data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
new_data_range[0] = first_bad[-1] + 1
print >> decilog, " changing DATA_RANGE= to", new_data_range
modify_xdsinp(xdsinp,
inp_params=[("JOB", "INIT"),
("DATA_RANGE",
"%d %d" % tuple(new_data_range))])
for f in xds_files.generated_by_INIT:
util.rotate_file(os.path.join(root, f), copy=False)
run_xds(wdir=root, show_progress=params.show_progress)
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1,
spots))
data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(
h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print >> decilog, "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (
iqr, q14, cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print >> decilog, "DEBUG:: keep_frames=", keep_frames
print >> decilog, "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(
lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(
x), cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"),
frame_selection=set(keep_frames))
# Indexing
if params.cell_prior.method == "use_first":
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
else:
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, "" # TODO indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.sgnum > 0:
# Check anyway
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(
xs_prior, xsxds, params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
if params.cell_prior.check:
print >> decilog, " Incompatible cell. Indexing failed."
return
else:
print >> decilog, " Warning: Incompatible cell."
elif params.cell_prior.method == "symm_constraint_only":
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Trying symmetry-constrained cell parameter:", cell
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
for f in xds_files.generated_by_IDXREF:
util.rotate_file(os.path.join(root, f),
copy=(f == "SPOT.XDS"))
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
# Check again
xsxds = XPARM(xparm).crystal_symmetry()
if not xsxds.unit_cell().is_similar_to(
xs_prior.unit_cell(), params.cell_prior.tol_length,
params.cell_prior.tol_angle):
print >> decilog, " Resulted in different cell. Indexing failed."
return
elif params.mode == "recycle":
print >> decilog, " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER",
"%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
pointless_integrate = {}
if params.use_pointless:
worker = Pointless()
pointless_integrate = worker.run_for_symm(
xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in pointless_integrate:
symm = pointless_integrate["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(symm.space_group(),
xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
else:
print >> decilog, " pointless failed."
flag_do_not_change_symm = False
if xs_prior and params.cell_prior.force:
modify_xdsinp(xdsinp,
inp_params=[("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum)])
flag_do_not_change_symm = True
elif params.cell_prior.method == "correct_only":
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(xs_prior, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is not None:
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Using given symmetry in CORRECT with symmetry constraints:", cell
modify_xdsinp(xdsinp,
inp_params=[
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
flag_do_not_change_symm = True
else:
print >> decilog, " Tried to use given symmetry in CORRECT, but cell in integration is incompatible."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xac_hkl):
print >> decilog, " CORRECT failed."
return
if not os.path.isfile(gxparm):
print >> decilog, " Refinement in CORRECT failed."
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(xac_hkl, os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
pointless_correct = worker.run_for_symm(
xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
pointless_best_symm = None
if "symm" in pointless_correct:
symm = pointless_correct["symm"]
need_rescale = False
if pointless_integrate.get("symm"):
symm_by_integrate = pointless_integrate["symm"]
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(),
symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
prob_integrate = pointless_integrate.get(
"laue_prob", float("nan"))
prob_correct = pointless_correct.get(
"laue_prob", float("nan"))
print >> decilog, " Prob(%s |INTEGRATE), Prob(%s |CORRECT) = %.4f, %.4f." % (
symm_by_integrate.space_group_info(),
symm.space_group_info(), prob_integrate,
prob_correct)
if prob_correct > prob_integrate:
need_rescale = True
pointless_best_symm = symm
else:
pointless_best_symm = symm_by_integrate
else:
need_rescale = True
pointless_best_symm = symm
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
if need_rescale and not flag_do_not_change_symm:
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)
])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(
xac_hkl,
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root,
show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
else:
print >> decilog, "ISa got worse= %.2f" % ISa
if pointless_best_symm:
xac_symm = XDS_ASCII(xac_hkl, read_data=False).symm
if not xtal.is_same_space_group_ignoring_enantiomorph(
xac_symm.space_group(),
pointless_best_symm.space_group()):
if xtal.is_same_laue_symmetry(
xac_symm.space_group(),
pointless_best_symm.space_group()):
tmp = "same Laue symmetry"
else:
tmp = "different Laue symmetry"
print >> decilog, "WARNING: symmetry in scaling is different from Pointless result (%s)." % tmp
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
except:
print >> decilog, traceback.format_exc()
finally:
print >> decilog, "\nxds_sequence finished at %s" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def run(params, out=sys.stdout):
cm = CheckMulti(topdir=params.topdir, xdsdirs=params.xdsdir, out=out)
cm.get_symms_from_xds_results()
cm.construct_graph(params.tol_length, params.tol_angle)
cm.group_xds_results()
print
ret = cm.grouped_dirs
if len(ret) == 0:
return cm
print >> out
print >> out, "About the largest group:"
for idx, wd in enumerate(ret[0]):
gxparm_xds = os.path.join(wd, "GXPARM.XDS")
correct_lp = os.path.join(wd, "CORRECT.LP")
print >> out, "%.3d %s" % (idx, os.path.relpath(wd, params.topdir)
if params.topdir is not None else wd),
if not os.path.isfile(gxparm_xds):
print >> out, "Unsuccessful"
continue
sg = sgtbx.space_group_info(XPARM(gxparm_xds).spacegroup)
clp = correctlp.CorrectLp(correct_lp)
if "all" in clp.table:
cmpl = clp.table["all"]["cmpl"][-1]
else:
cmpl = float("nan")
ISa = clp.a_b_ISa[-1]
print >> out, "%10s ISa=%5.2f Cmpl=%5.1f " % (sg, ISa, cmpl)
if params.do_pointless:
worker = Pointless()
files = map(lambda x: os.path.join(x, "INTEGRATE.HKL"), ret[0])
#print files
files = filter(lambda x: os.path.isfile(x), files)
print >> out, "\nRunning pointless for the largest member."
result = worker.run_for_symm(xdsin=files,
logout="pointless.log",
tolerance=10,
d_min=5)
if "symm" in result:
print >> out, " pointless suggested", result[
"symm"].space_group_info()
if 0:
import pylab
pos = nx.spring_layout(G)
#pos = nx.spectral_layout(G)
#pos = nx.circular_layout(G)
#nx.draw_networkx_nodes(G, pos, node_size = 100, nodelist=others, node_color = 'w')
nx.draw_networkx_nodes(G, pos, node_size=100, node_color='w')
nx.draw_networkx_edges(G, pos, width=1)
nx.draw_networkx_labels(G,
pos,
font_size=12,
font_family='sans-serif',
font_color='r')
pylab.xticks([])
pylab.yticks([])
pylab.savefig("network.png")
pylab.show()
return cm
def rescale_with_specified_symm(topdir,
dirs,
symms,
out,
sgnum=None,
reference_symm=None):
assert (sgnum, reference_symm).count(None) == 1
if sgnum is not None:
sgnum_laue = sgtbx.space_group_info(sgnum).group(
).build_derived_reflection_intensity_group(False).type().number()
matches = filter(
lambda x: x.reflection_intensity_symmetry(False).space_group_info(
).type().number() == sgnum_laue, symms)
matched_cells = numpy.array(
map(lambda x: x.unit_cell().parameters(), matches))
median_cell = map(lambda x: numpy.median(matched_cells[:, x]),
xrange(6))
reference_symm = crystal.symmetry(median_cell, sgnum)
else:
sgnum = reference_symm.space_group_info().type().number()
sgnum_laue = reference_symm.space_group(
).build_derived_reflection_intensity_group(False).type().number()
print >> out
print >> out, "Re-scaling with specified symmetry:", reference_symm.space_group_info(
).symbol_and_number()
print >> out, " reference cell:", reference_symm.unit_cell()
print >> out
print >> out
cells = {} # cell and file
for sym, wd in zip(symms, dirs):
print >> out, os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
continue
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (
os.path.basename(xac_file), xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(),
0.1, 10):
print >> out, " Already scaled with specified symmetry"
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()),
xac_file)
continue
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
20)
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = reference_symm.space_group().average_unit_cell(
xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT,
wdir=wd,
quiet=True)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f),
os.path.join(wd, f + "_rescale"))
revert_files(xds_files.generated_by_CORRECT,
bk_prefix,
wdir=wd,
quiet=True)
new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
print "OK:", cells[wd][0]
else:
print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
continue
return cells, reference_symm
def __init__(self, xparm_file):
XPARM.__init__(self, xparm_file)
self.bin = 1
self.set_bin()
def __init__(self, xparm_file):
XPARM.__init__(self, xparm_file)
self.bin = 1
self.set_bin()
def run(xds_inp):
xp = XPARM()
xp.set_info_from_xdsinp_or_inpstr(xdsinp=xds_inp)
d_map = numpy.zeros((xp.ny, xp.nx)) # resolution mapping
ed = numpy.zeros((3, 3))
ed[:, 0] = xp.X_axis
ed[:, 1] = xp.Y_axis
ed[:, 2] = numpy.cross(xp.X_axis, xp.Y_axis)
ed /= numpy.linalg.norm(ed, axis=0)
orgx, orgy = xp.origin
fs = xp.distance
qx, qy = xp.qx, xp.qy
org = -orgx * qx * ed[:, 0] - orgy * qy * ed[:, 1] + fs * ed[:, 2]
wavelen = xp.wavelength
s0 = xp.incident_beam / numpy.linalg.norm(xp.incident_beam) / wavelen
print "qx,qy=", qx, qy
print "s0=", s0
print "lambda=", wavelen
print "ORG=", org
print "ED="
print ed
ofs_pdb = open("detector_pos.pdb", "w")
ofs_pml = open("for_pymol.pml", "w")
for iseg, seg in enumerate(xp.segments):
print("Segment %d" % (iseg + 1))
eds = numpy.zeros((3, 3))
eds[:, 0] = seg.eds_x
eds[:, 1] = seg.eds_y
eds[:, 2] = numpy.cross(seg.eds_x, seg.eds_y)
eds /= numpy.linalg.norm(eds, axis=0)
edsl = numpy.dot(ed, eds)
ix, iy = numpy.meshgrid(range(seg.x1 - 1, seg.x2),
range(seg.y1 - 1, seg.y2))
tmp = numpy.zeros((ix.shape[0], ix.shape[1], 3))
for i in range(3):
tmp[:, :, i] = qx * (ix - seg.orgxs) * edsl[i, 0] + qy * (
iy - seg.orgys) * edsl[i, 1] + seg.fs * edsl[i, 2] + org[i]
write_atom(ofs_pdb, tmp, iseg)
ofs_pml.write("bond resi %d and name C1, resi %d and name C2\n" %
(iseg, iseg))
ofs_pml.write("bond resi %d and name C2, resi %d and name C4\n" %
(iseg, iseg))
ofs_pml.write("bond resi %d and name C3, resi %d and name C4\n" %
(iseg, iseg))
ofs_pml.write("bond resi %d and name C3, resi %d and name C1\n" %
(iseg, iseg))
tmpdenom = numpy.linalg.norm(tmp, axis=2) * wavelen
for i in range(3):
tmp[:, :, i] /= tmpdenom
s1 = tmp
s = s1 - s0
d_map[iy, ix] = 1. / numpy.linalg.norm(s, axis=2)
"""
for ix in range(seg.x1-1, seg.x2):
for iy in range(seg.y1-1, seg.y2):
tmp = (qx*(ix-seg.orgxs)*edsl[0,0]+qy*(iy-seg.orgys)*edsl[0,1]+seg.fs*edsl[0,2]+orgx,
qx*(ix-seg.orgxs)*edsl[1,0]+qy*(iy-seg.orgys)*edsl[1,1]+seg.fs*edsl[1,2]+orgy,
qx*(ix-seg.orgxs)*edsl[2,0]+qy*(iy-seg.orgys)*edsl[2,1]+seg.fs*edsl[2,2]+fs)
s1 = tmp/numpy.linalg.norm(tmp)/wavelen
s = s1-s0
d_map[iy,ix] = numpy.linalg.norm(s)
"""
ofs_pml.write("pseudoatom org, pos=(0,0,0)\n")
ofs_pml.write("pseudoatom s0, pos=(%f,%f,%f)\n" % tuple(s0 * 100))
ofs_pml.write("as spheres, org s0\n")
ofs_pml.write("color red, s0\n")
ofs_pml.write("distance selection1=org, selection2=s0\n")
ofs_pml.write("set sphere_scale, 10\n")
data = (d_map * 1000).astype(numpy.int32)
cbf.save_numpy_data_as_cbf(data.flatten(), data.shape[1], data.shape[0],
"d_map", "d_map_x1000.cbf")
def set_xparm(self, xparm_in):
xparm = XPARM(xparm_in)
self.calc_d = lambda x, y: xparm.wavelength / 2. / math.sin(
0.5 * math.atan(
math.sqrt((x - xparm.origin[0])**2 + (y - xparm.origin[1])**2)
* xparm.qx / abs(xparm.distance)))
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in+ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir): os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >>decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >>decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True, anomalous=params.anomalous,
spot_range=None, minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx, orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f"%params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f"%params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d"%params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f"%params.sepmin),
("CLUSTER_RADIUS", "%.2f"%params.cluster_radius),
("INDEX_ERROR", "%.4f"%params.index_error),
("INDEX_MAGNITUDE", "%d"%params.index_magnitude),
("INDEX_QUALITY", "%.2f"%params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
("VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels))
])
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >>decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f"%x, params.cell))),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >>decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min)),
])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell)))]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum)] + tmp)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f+".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(f, read_columns=["IOBS","SIGMA","XCAL","YCAL","RLP","PEAK","MAXC"])
pickle.dump(inhkl, open(f+".pkl", "w"), -1)
except ProcFailed, e:
print >>decilog, "Processing failed: %s" % e.message
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in + ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >> decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >> decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(
img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True,
anomalous=params.anomalous,
spot_range=None,
minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx,
orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance,
)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f" % params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f" % params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d" % params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f" % params.sepmin),
("CLUSTER_RADIUS", "%.2f" % params.cluster_radius),
("INDEX_ERROR", "%.4f" % params.index_error),
("INDEX_MAGNITUDE", "%d" % params.index_magnitude),
("INDEX_QUALITY", "%.2f" % params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
(
"VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS",
"%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels),
),
],
)
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
flag_try_cell_hint = True
if flag_try_cell_hint:
print >> decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
],
)
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0")])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >> decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
],
)
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min))])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE")])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [
("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
]
+ tmp,
)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >> decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle:
x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f + ".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >> decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(
f, read_columns=["IOBS", "SIGMA", "XCAL", "YCAL", "RLP", "PEAK", "MAXC"]
)
pickle.dump(inhkl, open(f + ".pkl", "w"), -1)
except ProcFailed, e:
print >> decilog, "Processing failed: %s" % e.message
