def add_proc_result(self, key, xdsdir):
if key in self.G: return #G.remove_node(key)
p1cell, symm = self.get_p1cell_and_symm(xdsdir)
if None in (p1cell, symm): return
self.p1cells[key] = p1cell
self.dirs[key] = xdsdir
self.symms[key] = symm
connected_nodes = []
for node in self.G.nodes_iter():
other_cell = self.p1cells[node]
if other_cell.is_similar_to(p1cell, self.tol_length,
self.tol_angle):
connected_nodes.append(node)
else:
cosets = reindex.reindexing_operators(
crystal.symmetry(other_cell,
1), crystal.symmetry(p1cell, 1),
self.tol_length, self.tol_angle)
if cosets.double_cosets is not None:
self.cbops[(node, key)] = cosets.combined_cb_ops()[0]
print p1cell, other_cell, self.cbops[(
node, key)], other_cell.change_basis(self.cbops[(node,
key)])
connected_nodes.append(node)
# Add nodes and edges
self.G.add_node(key)
for node in connected_nodes:
self.G.add_edge(node, key)
def find_reindex_ops(self):
symm = self.arrays[0].crystal_symmetry()
cosets = reindex.reindexing_operators(symm,
symm,
max_delta=self.max_delta)
reidx_ops = cosets.combined_cb_ops()
return reidx_ops
def reindex_with_specified_symm(topdir, reference_symm, dirs, out):
print >>out
print >>out, "Re-index to specified symmetry:"
reference_symm.show_summary(out, " ")
print >>out
print >>out
cells = {} # cell and file
sgnum_laue = reference_symm.space_group().build_derived_reflection_intensity_group(False).type().number()
for wd in dirs:
print >>out, "%s:" % os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >>out, "Can't find XDS_ASCII file in %s" % wd
continue
if xac_file.endswith(".org"): xac_file_org, xac_file = xac_file, xac_file[:-4]
else: xac_file_org = xac_file+".org"
if not os.path.isfile(xac_file_org):
os.rename(xac_file, xac_file_org)
xac = XDS_ASCII(xac_file_org, read_data=False)
print >>out, "%s %s (%s)" % (os.path.basename(xac_file), xac.symm.space_group_info(),
",".join(map(lambda x: "%.2f"%x, xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1, 10):
print >>out, " Already scaled with specified symmetry"
os.rename(xac_file_org, xac_file) # rename back
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()), xac_file)
continue
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)
if len(cosets.combined_cb_ops())==0:
print >>out, "Can't find operator:"
xac.symm.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = xac.write_reindexed(op=cosets.combined_cb_ops()[0],
space_group=reference_symm.space_group(),
hklout=xac_file)
cells[wd] = (numpy.array(newcell.parameters()), xac_file)
newcell = " ".join(map(lambda x: "%.3f"%x, newcell.parameters()))
print >>out, " Reindexed to transformed cell: %s with %s" % (newcell, cosets.combined_cb_ops()[0].as_hkl())
return cells
def __init__(self,
set_a,
set_b,
out=None,
relative_length_tolerance=0.05,
absolute_angle_tolerance=3.0,
lattice_symmetry_max_delta=3.0,
file_name=None):
self.relative_length_tolerance = relative_length_tolerance
self.absolute_angle_tolerance = absolute_angle_tolerance
self.lattice_symmetry_max_delta = lattice_symmetry_max_delta
self.out = out
if self.out is None:
self.out = sys.stdout
self.file_name = file_name
## make deep copy for safety, and make it non-anomalous
self.set_b_ori = set_b.deep_copy().set_observation_type(set_b)
self.set_a = set_a.deep_copy().set_observation_type(
set_a).average_bijvoet_mates()
self.set_b = set_b.deep_copy().set_observation_type(
set_b).average_bijvoet_mates()
self.set_a_xs = crystal.symmetry(unit_cell=self.set_a.unit_cell(),
space_group=self.set_a.space_group())
self.set_b_xs = crystal.symmetry(unit_cell=self.set_b.unit_cell(),
space_group=self.set_b.space_group())
self.cosets = reindex.reindexing_operators(
xs1=self.set_a_xs,
xs2=self.set_b_xs,
relative_length_tolerance=self.relative_length_tolerance,
absolute_angle_tolerance=self.absolute_angle_tolerance,
max_delta=self.lattice_symmetry_max_delta,
anomalous_flag=self.set_a.anomalous_flag(),
out=self.out)
self.set_a_to_niggli = self.cosets.cb_op_to_n_1
self.set_b_to_niggli = self.cosets.cb_op_to_n_2
self.set_a_nig = self.set_a.change_basis(
self.set_a_to_niggli).map_to_asu()
self.set_b_nig = self.set_b.change_basis(
self.set_b_to_niggli).map_to_asu()
self.ops_in_niggli_setting = self.cosets.cb_ops_in_niggli_setting()
self.nice_cb_ops = self.cosets.combined_cb_ops()
self.cc_values = []
self.matches = []
if len(self.nice_cb_ops) > 0:
self.analyse()
self.select_and_transform()
else:
print("No simple reindexing relation found between unit cells",
file=self.out)
def check_reidx(params, xs):
assert params.checkcell.check and params.sgnum > 0
xsref = crystal.symmetry(params.cell, params.sgnum)
cosets = reindex.reindexing_operators(xsref, xs,
params.checkcell.tol_length, params.checkcell.tol_angle)
if cosets.double_cosets is None:
return None
return cosets.combined_cb_ops()[0]
def get_angles(ib, a_axis, b_axis, c_axis, xs, ref_xs, filename):
ib = ib / numpy.linalg.norm(ib)
if ref_xs:
cosets = reindex.reindexing_operators(ref_xs, xs, 0.2, 20)
if len(cosets.combined_cb_ops()) == 0:
print "Error: cannot find reindex operator"
return
op = cosets.combined_cb_ops()[0]
m = numpy.array(op.c_inv().r().as_double()).reshape(3, 3).transpose()
transformed = numpy.dot(m, numpy.array([a_axis, b_axis, c_axis]))
a, b, c = transformed[0, :], transformed[1, :], transformed[2, :]
sg = ref_xs.space_group()
print "Reading %s (%s; reindex: %s)" % (filename, sg.info(), op)
else:
a, b, c = a_axis, b_axis, c_axis
sg = xs.space_group()
print "Reading %s (%s)" % (filename, sg.info())
laue = sg.build_derived_reflection_intensity_group(
False).build_derived_point_group()
astar, bstar, cstar = abc_convert_real_reciprocal(a, b, c)
bstar_e = bstar / numpy.linalg.norm(bstar)
cstar_e = cstar / numpy.linalg.norm(cstar)
# incident_beam along a,b,c
xyz = (numpy.dot(ib, a) / numpy.linalg.norm(a),
numpy.dot(ib, b) / numpy.linalg.norm(b),
numpy.dot(ib, c) / numpy.linalg.norm(c))
angles = []
#print "Replicating vectors using %2d operators" % len(laue.all_ops())
# replicate vectors
for op in laue.all_ops():
vec_elem = op * xyz
vec = vec_elem[0] * a + vec_elem[1] * b + vec_elem[2] * c
veclen = numpy.linalg.norm(vec)
theta = numpy.arccos(numpy.dot(vec, cstar_e) / veclen)
if theta > numpy.pi / 2: continue
vec_on_ab = vec - cstar_e * veclen * numpy.cos(theta)
phi = numpy.arccos(
numpy.dot(vec_on_ab, a) / numpy.linalg.norm(a) /
numpy.linalg.norm(vec_on_ab))
if numpy.dot(vec_on_ab, bstar_e) < 0: phi = -phi
angles.append((theta, phi))
return angles
def __init__(self,
set_a,
set_b,
out=None,
relative_length_tolerance=0.05,
absolute_angle_tolerance=3.0,
lattice_symmetry_max_delta=3.0,
file_name=None):
self.relative_length_tolerance=relative_length_tolerance
self.absolute_angle_tolerance=absolute_angle_tolerance
self.lattice_symmetry_max_delta=lattice_symmetry_max_delta
self.out = out
if self.out is None:
self.out = sys.stdout
self.file_name=file_name
## make deep copy for safety, and make it non-anomalous
self.set_b_ori = set_b.deep_copy().set_observation_type( set_b )
self.set_a = set_a.deep_copy().set_observation_type( set_a ).average_bijvoet_mates()
self.set_b = set_b.deep_copy().set_observation_type( set_b ).average_bijvoet_mates()
self.set_a_xs = crystal.symmetry( unit_cell=self.set_a.unit_cell(),
space_group=self.set_a.space_group() )
self.set_b_xs = crystal.symmetry( unit_cell=self.set_b.unit_cell(),
space_group=self.set_b.space_group() )
self.cosets = reindex.reindexing_operators(
xs1=self.set_a_xs,
xs2=self.set_b_xs,
relative_length_tolerance=self.relative_length_tolerance,
absolute_angle_tolerance=self.absolute_angle_tolerance,
max_delta=self.lattice_symmetry_max_delta,
anomalous_flag=self.set_a.anomalous_flag(),
out=self.out
)
self.set_a_to_niggli = self.cosets.cb_op_to_n_1
self.set_b_to_niggli = self.cosets.cb_op_to_n_2
self.set_a_nig = self.set_a.change_basis( self.set_a_to_niggli ).map_to_asu()
self.set_b_nig = self.set_b.change_basis( self.set_b_to_niggli ).map_to_asu()
self.ops_in_niggli_setting = self.cosets.cb_ops_in_niggli_setting()
self.nice_cb_ops = self.cosets.combined_cb_ops()
self.cc_values = []
self.matches = []
if len(self.nice_cb_ops)>0:
self.analyse()
self.select_and_transform()
else:
print >> self.out, "No simple reindexing relation found between unit cells"
def get_angles(ib, a_axis, b_axis, c_axis, xs, ref_xs, filename):
ib = ib / numpy.linalg.norm(ib)
if ref_xs:
cosets = reindex.reindexing_operators(ref_xs, xs, 0.2, 20)
if len(cosets.combined_cb_ops())==0:
print "Error: cannot find reindex operator"
return
op = cosets.combined_cb_ops()[0]
m = numpy.array(op.c_inv().r().as_double()).reshape(3,3).transpose()
transformed = numpy.dot(m, numpy.array([a_axis,b_axis,c_axis]))
a, b, c = transformed[0,:], transformed[1,:], transformed[2,:]
sg = ref_xs.space_group()
print "Reading %s (%s; reindex: %s)" % (filename, sg.info(), op)
else:
a, b, c = a_axis, b_axis, c_axis
sg = xs.space_group()
print "Reading %s (%s)" % (filename, sg.info())
laue = sg.build_derived_reflection_intensity_group(False)
astar, bstar, cstar = abc_convert_real_reciprocal(a,b,c)
bstar_e = bstar / numpy.linalg.norm(bstar)
cstar_e = cstar / numpy.linalg.norm(cstar)
# incident_beam along a,b,c
xyz = (numpy.dot(ib, a)/numpy.linalg.norm(a),
numpy.dot(ib, b)/numpy.linalg.norm(b),
numpy.dot(ib, c)/numpy.linalg.norm(c))
angles = []
#print "Replicating vectors using %2d operators" % len(laue.all_ops())
# replicate vectors
for op in laue.all_ops():
vec_elem = op * xyz
vec = vec_elem[0] * a + vec_elem[1] * b + vec_elem[2] * c
veclen = numpy.linalg.norm(vec)
theta = numpy.arccos(numpy.dot(vec, cstar_e)/veclen)
if theta > numpy.pi/2: continue
vec_on_ab = vec - cstar_e * veclen*numpy.cos(theta)
phi = numpy.arccos(numpy.dot(vec_on_ab, a)/numpy.linalg.norm(a)/numpy.linalg.norm(vec_on_ab))
if numpy.dot(vec_on_ab, bstar_e) < 0: phi = - phi
angles.append((theta, phi))
return angles
def unique_reindexing_operators(self, other, relative_length_tolerance, absolute_angle_tolerance):
# make crystal symmetries
self_xs = crystal.symmetry(unit_cell=self.input.unit_cell(), space_group=self.input.space_group())
other_xs = crystal.symmetry(unit_cell=other.input.unit_cell(), space_group=other.input.space_group())
# some downstream routines expect things to be in minimum cell
self_xs = self_xs.change_basis(self.change_of_basis_op_to_minimum_cell)
other_xs = other_xs.change_basis(other.change_of_basis_op_to_minimum_cell)
double_cosets = reindex.reindexing_operators(
self_xs, other_xs, relative_length_tolerance, absolute_angle_tolerance, self.input.anomalous_flag()
)
result = double_cosets.combined_cb_ops()
return result
def unique_reindexing_operators(self, other, relative_length_tolerance,
absolute_angle_tolerance):
#make crystal symmetries
self_xs = crystal.symmetry(unit_cell=self.input.unit_cell(),
space_group=self.input.space_group())
other_xs = crystal.symmetry(unit_cell=other.input.unit_cell(),
space_group=other.input.space_group())
# some downstream routines expect things to be in minimum cell
self_xs = self_xs.change_basis(self.change_of_basis_op_to_minimum_cell)
other_xs = other_xs.change_basis(
other.change_of_basis_op_to_minimum_cell)
double_cosets = reindex.reindexing_operators(
self_xs, other_xs, relative_length_tolerance,
absolute_angle_tolerance, self.input.anomalous_flag())
result = double_cosets.combined_cb_ops()
return result
def construct_graph(self, tol_length, tol_angle):
self.G = nx.Graph()
for i in xrange(len(self.p1cells)):
self.G.add_node(i)
for i in xrange(len(self.p1cells)-1):
for j in xrange(i+1, len(self.p1cells)):
cell_i = self.p1cells[i]
cell_j = self.p1cells[j]
if cell_i.is_similar_to(cell_j, tol_length, tol_angle):
self.G.add_edge(i, j)
else:
cosets = reindex.reindexing_operators(crystal.symmetry(cell_i, 1),
crystal.symmetry(cell_j, 1),
tol_length, tol_angle)
self.cosets[(i,j)] = cosets
if cosets.double_cosets is not None:
self.G.add_edge(i, j)
def construct_graph(self, tol_length, tol_angle):
self.G = nx.Graph()
for i in xrange(len(self.p1cells)):
self.G.add_node(i)
for i in xrange(len(self.p1cells) - 1):
for j in xrange(i + 1, len(self.p1cells)):
cell_i = self.p1cells[i]
cell_j = self.p1cells[j]
if cell_i.is_similar_to(cell_j, tol_length, tol_angle):
self.G.add_edge(i, j)
else:
cosets = reindex.reindexing_operators(
crystal.symmetry(cell_i, 1),
crystal.symmetry(cell_j, 1), tol_length, tol_angle)
self.cosets[(i, j)] = cosets
if cosets.double_cosets is not None:
print cell_i, cell_j, cosets.combined_cb_ops()[0]
self.G.add_edge(i, j)
def try_indexing_hard(wdir, show_progress, decilog,
known_sgnum=None, known_cell=None, tol_length=None, tol_angle=None):
idxref_lp = os.path.join(wdir, "IDXREF.LP")
xdsinp = os.path.join(wdir, "XDS.INP")
lp_org = idxreflp.IdxrefLp(idxref_lp)
if lp_org.is_cell_maybe_half():
backup_needed = ("XDS.INP",) + xds_files.generated_by_IDXREF
print >>decilog, " !! Cell may be halved. Trying doubled cell."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
cell = lp_org.deduce_correct_cell_based_on_integerness()
cell = " ".join(map(lambda x:"%.2f"%x, cell.parameters()))
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "1"),
("UNIT_CELL_CONSTANTS", cell)
])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
print >>decilog, " .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SEPMIN", "4"),
("CLUSTER_RADIUS", "2")
])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
print >>decilog, " .. not solved. Give up."
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
else:
print >>decilog, " Now OK."
remove_backups(backup_needed, bk_prefix, wdir=wdir)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
# If Cell hint exists, try to use it..
if known_sgnum > 0:
flag_try_cell_hint = False
xparm = os.path.join(wdir, "XPARM.XDS")
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(known_cell, known_sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
tol_length, tol_angle)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >>decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f"%x, known_cell))),
("SPACE_GROUP_NUMBER", "%d"%known_sgnum),
])
run_xds(wdir=wdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
decilog = multi_out()
decilog.register("log", open(os.path.join(root, "decision.log"), "a"), atexit_send_to=None)
print >>decilog, "xds_sequence started at %s in %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode=="initial" and params.resume and os.path.isfile(correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp, inp_params=[("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# To Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root, params.show_progress, decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >>decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell, params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length, params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >>decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS", "plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm+".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp, os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >>decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP",), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x:"%.3f"%x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d"%xparm_obj.spacegroup),])
print >>decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP",):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff+"_noscale")
revert_files(("XDS.INP",), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >>decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info()
sgnum = symm.space_group_info().type().number()
cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >>decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >>decilog, " Scaling failed."
return
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
print >>decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
print >>decilog, " (Original files are saved as *_fullres.*)"
else:
print >>decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(symm_by_integrate.space_group(), symm.space_group()):
print >>decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (symm.space_group_info(), symm_by_integrate.space_group_info())
need_rescale = True
else:
print >>decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info()
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL","CORRECT_fullres.LP",
"merging_stats.pkl","merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
print >>decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
print >>decilog, " (Original files are saved as *_fullres.*)"
else:
print >>decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP", "XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >>decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa!=last_ISa: # if improved or last_ISa is nan
print >>decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >>decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)): os.remove(os.path.join(root, f))
revert_files(backup_needed, bk_prefix, wdir=root, quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >>decilog, "xds_sequence finished at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
decilog.close()
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
])
if params.mode == "initial":
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1, spots))
data_range = map(int, xdsinp_dict["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (iqr, q14,
cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print "DEBUG:: keep_frames=", keep_frames
print "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(x),
cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"), frame_selection=set(keep_frames))
# Indexing
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print # indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.check and params.cell_prior.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
print >> decilog, " Incompatible cell. Indexing failed."
return
elif params.mode == "recycle":
print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x, xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER", "%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
symm_by_integrate = None
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=integrate_hkl,
logout=os.path.join(
root, "pointless_integrate.log"))
if "symm" in result:
symm = result["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
symm_by_integrate = symm
else:
print >> decilog, " pointless failed."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(gxparm):
print >> decilog, " Scaling failed."
return
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp,
check_valid=True)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
result = worker.run_for_symm(xdsin=xac_hkl,
logout=os.path.join(
root, "pointless_correct.log"))
if "symm" in result:
symm = result["symm"]
need_rescale = False
if symm_by_integrate is not None:
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(), symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
need_rescale = True
else:
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
need_rescale = True
if need_rescale:
# make backup, and do correct and compare ISa
# if ISa got worse, revert the result.
backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL",
"CORRECT_fullres.LP", "merging_stats.pkl",
"merging_stats.log")
backup_needed += xds_files.generated_by_CORRECT
bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(os.path.join(root, "XDS_ASCII.HKL"),
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
remove_backups(backup_needed, bk_prefix, wdir=root)
else:
print >> decilog, "ISa got worse= %.2f" % ISa
for f in backup_needed:
if os.path.isfile(os.path.join(root, f)):
os.remove(os.path.join(root, f))
revert_files(backup_needed,
bk_prefix,
wdir=root,
quiet=True)
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
print >> decilog, "xds_sequence finished at %s\n" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def try_indexing_hard(wdir,
show_progress,
decilog,
known_sgnum=None,
known_cell=None,
tol_length=None,
tol_angle=None):
idxref_lp = os.path.join(wdir, "IDXREF.LP")
xdsinp = os.path.join(wdir, "XDS.INP")
lp_org = idxreflp.IdxrefLp(idxref_lp)
if lp_org.is_cell_maybe_half():
backup_needed = ("XDS.INP", ) + xds_files.generated_by_IDXREF
print >> decilog, " !! Cell may be halved. Trying doubled cell."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
cell = lp_org.deduce_correct_cell_based_on_integerness()
cell = " ".join(map(lambda x: "%.2f" % x, cell.parameters()))
modify_xdsinp(xdsinp,
inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "1"),
("UNIT_CELL_CONSTANTS", cell)])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
print >> decilog, " .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)
modify_xdsinp(xdsinp,
inp_params=[("JOB", "IDXREF"), ("SEPMIN", "4"),
("CLUSTER_RADIUS", "2")])
run_xds(wdir=wdir, show_progress=show_progress)
if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
print >> decilog, " .. not solved. Give up."
revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
else:
print >> decilog, " Now OK."
remove_backups(backup_needed, bk_prefix, wdir=wdir)
modify_xdsinp(xdsinp, inp_params=[
("SPACE_GROUP_NUMBER", "0"),
])
# If Cell hint exists, try to use it..
if known_sgnum > 0:
flag_try_cell_hint = False
xparm = os.path.join(wdir, "XPARM.XDS")
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
xsref = crystal.symmetry(known_cell, known_sgnum)
cosets = reindex.reindexing_operators(xsref, xsxds, tol_length,
tol_angle)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >> decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f" % x,
known_cell))),
("SPACE_GROUP_NUMBER", "%d" % known_sgnum),
])
run_xds(wdir=wdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[
("SPACE_GROUP_NUMBER", "0"),
])
#ksb.debug_write_mtz()
#ksb.modify_xds_ascii_files()
quit()
arrays = []
for f in xac_files:
print "Reading", f
xac = XDS_ASCII(f, i_only=True)
xac.remove_rejected()
a = xac.i_obs().resolution_filter(d_min=3)
a = a.merge_equivalents(use_internal_variance=False).array()
arrays.append(a)
symm = arrays[0].crystal_symmetry()
cosets = reindex.reindexing_operators(symm, symm)
reidx_ops = cosets.combined_cb_ops()
reidx_ops.sort(key=lambda x: not x.is_identity_op())
print " Possible reindex operators:", map(lambda x: str(x.as_hkl()), reidx_ops)
determined = set([0,])
old_ops = map(lambda x:0, xrange(len(arrays)))
for ncycle in xrange(100): # max cycle
new_ops = map(lambda x:0, xrange(len(arrays)))
for i in xrange(len(arrays)):
cc_list = []
a = arrays[i]
for j, op in enumerate(reidx_ops):
tmp = a.customized_copy(indices=op.apply(a.indices())).map_to_asu()
for ref in determined:
def prep_merging(self,
workdir,
group,
symmidx=None,
reference_symm=None,
topdir=None,
cell_method="reindex",
nproc=1,
prep_dials_files=True,
into_workdir=True):
assert (symmidx, reference_symm).count(None) == 1
from yamtbx.util.xtal import format_unit_cell
from cctbx.crystal import reindex
cm = self.cell_graph
prep_log_out = multi_out()
prep_log_out.register("log",
open(os.path.join(workdir, "prep_merge.log"),
"w"),
atexit_send_to=None)
prep_log_out.register("stdout", sys.stdout)
prep_log_out.write(self.log_buffer)
prep_log_out.flush()
if reference_symm is None:
reference_symm = cm.get_reference_symm(group - 1, symmidx)
prep_log_out.write("\n\ngroup_choice= %d, symmetry= %s (%s)\n" %
(group, reference_symm.space_group_info(),
format_unit_cell(reference_symm.unit_cell())))
prep_log_out.flush()
# Scale with specified symmetry
symms = map(lambda i: cm.symms[i], cm.groups[group - 1])
dirs = map(lambda i: cm.dirs[i], cm.groups[group - 1])
copyto_root = os.path.join(workdir,
"input_files") if into_workdir else None
if not topdir: topdir = os.path.dirname(os.path.commonprefix(dirs))
if cell_method == "reindex":
self.cell_and_files = reindex_with_specified_symm(
topdir,
reference_symm,
dirs,
out=prep_log_out,
nproc=nproc,
prep_dials_files=prep_dials_files,
copyto_root=copyto_root)
elif cell_method == "refine":
self.cell_and_files, reference_symm = rescale_with_specified_symm(
topdir,
dirs,
symms,
reference_symm=reference_symm,
out=prep_log_out,
nproc=nproc,
prep_dials_files=prep_dials_files,
copyto_root=copyto_root)
else:
raise "Don't know this choice: %s" % cell_method
prep_log_out.flush()
cosets = reindex.reindexing_operators(reference_symm,
reference_symm,
max_delta=5)
reidx_ops = cosets.combined_cb_ops()
print >> prep_log_out, "\nReference symmetry:", reference_symm.unit_cell(
), reference_symm.space_group_info().symbol_and_number()
msg_reindex = "\n"
if len(reidx_ops) > 1:
msg_reindex += "!! ATTENTION !! Reindex operators found. You may need to reindex some files before merging.\n"
for rop in reidx_ops:
msg_reindex += " operator: %-16s Cell: (%s)\n" % (
rop.as_hkl(),
format_unit_cell(
reference_symm.unit_cell().change_basis(rop)))
msg_reindex += "Try kamo.resolve_indexing_ambiguity command before merging!!"
else:
msg_reindex += "No reindex operators found. No need to run kamo.resolve_indexing_ambiguity."
print >> prep_log_out, "%s\n\n" % msg_reindex
prep_log_out.close()
# Make list for merging
ofs_lst = open(os.path.join(workdir, "formerge.lst"), "w")
ofs_dat = open(os.path.join(workdir, "cells.dat"), "w")
ofs_dat.write("file a b c al be ga\n")
for wd in sorted(self.cell_and_files):
cell, xas = self.cell_and_files[wd]
ofs_lst.write(xas + "\n")
ofs_dat.write(xas + " " +
" ".join(map(lambda x: "%7.3f" % x, cell)) + "\n")
ofs_dat.close()
ofs_lst.close()
return msg_reindex, reidx_ops
def rescale_with_specified_symm(topdir,
dirs,
symms,
out,
sgnum=None,
reference_symm=None):
assert (sgnum, reference_symm).count(None) == 1
if sgnum is not None:
sgnum_laue = sgtbx.space_group_info(sgnum).group(
).build_derived_reflection_intensity_group(False).type().number()
matches = filter(
lambda x: x.reflection_intensity_symmetry(False).space_group_info(
).type().number() == sgnum_laue, symms)
matched_cells = numpy.array(
map(lambda x: x.unit_cell().parameters(), matches))
median_cell = map(lambda x: numpy.median(matched_cells[:, x]),
xrange(6))
reference_symm = crystal.symmetry(median_cell, sgnum)
else:
sgnum = reference_symm.space_group_info().type().number()
sgnum_laue = reference_symm.space_group(
).build_derived_reflection_intensity_group(False).type().number()
print >> out
print >> out, "Re-scaling with specified symmetry:", reference_symm.space_group_info(
).symbol_and_number()
print >> out, " reference cell:", reference_symm.unit_cell()
print >> out
print >> out
cells = {} # cell and file
for sym, wd in zip(symms, dirs):
print >> out, os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
continue
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (
os.path.basename(xac_file), xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(),
0.1, 10):
print >> out, " Already scaled with specified symmetry"
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()),
xac_file)
continue
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
20)
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = reference_symm.space_group().average_unit_cell(
xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT,
wdir=wd,
quiet=True)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f),
os.path.join(wd, f + "_rescale"))
revert_files(xds_files.generated_by_CORRECT,
bk_prefix,
wdir=wd,
quiet=True)
new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
print "OK:", cells[wd][0]
else:
print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
continue
return cells, reference_symm
def check_cell(params, xs):
assert params.checkcell.check and params.sgnum > 0
xsref = crystal.symmetry(params.cell, params.sgnum)
cosets = reindex.reindexing_operators(xsref, xs, params.checkcell.tol_length, params.checkcell.tol_angle)
return cosets
def run(params):
out = multi_out()
out.register("log",
open(
os.path.join(os.path.dirname(params.lstout),
"multi_prep_merging.log"), "w"),
atexit_send_to=None)
out.register("stdout", sys.stdout)
cell_params = libtbx.phil.parse(
input_string=multi_check_cell_consistency.master_params_str).extract()
cell_params.topdir = params.topdir
cm = multi_check_cell_consistency.run(cell_params, out=out)
if len(cm.groups) == 0:
print "Oh, no. No data."
return
if params.space_group is not None and params.group_choice is None:
params.group_choice = 1 # maybe the largest group.
if params.group_choice < 1 or len(cm.groups) < params.group_choice:
print "Invalid group_choice=. Should be in 1..%d" % len(cm.groups)
return
possible_sgs = set(
map(lambda x: cm.symms[x].space_group(),
cm.groups[params.group_choice - 1]))
if params.space_group is None:
print "Please specify space_group=. The possible choice is:"
print "\n".join(
map(lambda x: x.info().symbol_and_number(), possible_sgs))
return
try:
if sgtbx.space_group_info(
params.space_group).group() not in possible_sgs:
print "Invalid space group choice. The possible choice is:"
print "\n".join(
map(lambda x: x.info().symbol_and_number(), possible_sgs))
return
except RuntimeError:
print "Invalid space group name or number (%s)" % params.space_group
return
symms = map(lambda i: cm.symms[i], cm.groups[params.group_choice - 1])
dirs = map(lambda i: cm.dirs[i], cm.groups[params.group_choice - 1])
sgnum = sgtbx.space_group_info(params.space_group).group().type().number()
# 1. Scale with specified symmetry
rescale_with_specified_symm(params.topdir, dirs, symms, out, sgnum=sgnum)
# 2. Resolve reindexing problem
# TODO: reconstruct unit cell by averaging
reference_symm = filter(
lambda x: x.reflection_intensity_symmetry(False).space_group_info().
type().number() == sgnum_laue, symms)[0]
cosets = reindex.reindexing_operators(reference_symm, reference_symm)
reidx_ops = cosets.combined_cb_ops()
print " Reference symmetry:", reference_symm.unit_cell(
), reference_symm.space_group_info().symbol_and_number(),
print " Possible reindex operators:", map(lambda x: str(x.as_hkl()),
reidx_ops)
if len(reidx_ops) == 1:
print " No reindexing problem exists."
elif params.reference_for_reindex is not None:
resolve_indexing_ambiguity_using_reference(
dirs, reidx_ops, params.reference_for_reindex)
else:
resolve_indexing_ambiguity(dirs, reidx_ops)
ofs = open(params.lstout, "w")
for wd in dirs:
xas_full = os.path.join(wd, "XDS_ASCII_fullres.HKL")
if os.path.isfile(xas_full): # if _fullres.HKL exists, use it.
ofs.write("%s\n" % xas_full)
else:
ofs.write("%s\n" % os.path.join(wd, "XDS_ASCII.HKL"))
def run(params):
out = multi_out()
out.register("log", open(os.path.join(os.path.dirname(params.lstout), "multi_prep_merging.log"), "w"), atexit_send_to=None)
out.register("stdout", sys.stdout)
cell_params = libtbx.phil.parse(input_string=multi_check_cell_consistency.master_params_str).extract()
cell_params.topdir = params.topdir
cm = multi_check_cell_consistency.run(cell_params, out=out)
if len(cm.groups) == 0:
print "Oh, no. No data."
return
if params.space_group is not None and params.group_choice is None:
params.group_choice = 1 # maybe the largest group.
if params.group_choice < 1 or len(cm.groups) < params.group_choice:
print "Invalid group_choice=. Should be in 1..%d" % len(cm.groups)
return
possible_sgs = set(map(lambda x: cm.symms[x].space_group(), cm.groups[params.group_choice-1]))
if params.space_group is None:
print "Please specify space_group=. The possible choice is:"
print "\n".join(map(lambda x: x.info().symbol_and_number(), possible_sgs))
return
try:
if sgtbx.space_group_info(params.space_group).group() not in possible_sgs:
print "Invalid space group choice. The possible choice is:"
print "\n".join(map(lambda x: x.info().symbol_and_number(), possible_sgs))
return
except RuntimeError:
print "Invalid space group name or number (%s)" % params.space_group
return
symms = map(lambda i: cm.symms[i], cm.groups[params.group_choice-1])
dirs = map(lambda i: cm.dirs[i], cm.groups[params.group_choice-1])
sgnum = sgtbx.space_group_info(params.space_group).group().type().number()
# 1. Scale with specified symmetry
rescale_with_specified_symm(params.topdir, dirs, symms, out, sgnum=sgnum)
# 2. Resolve reindexing problem
# TODO: reconstruct unit cell by averaging
reference_symm = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)[0]
cosets = reindex.reindexing_operators(reference_symm, reference_symm)
reidx_ops = cosets.combined_cb_ops()
print " Reference symmetry:", reference_symm.unit_cell(), reference_symm.space_group_info().symbol_and_number(),
print " Possible reindex operators:", map(lambda x: str(x.as_hkl()), reidx_ops)
if len(reidx_ops) == 1:
print " No reindexing problem exists."
elif params.reference_for_reindex is not None:
resolve_indexing_ambiguity_using_reference(dirs, reidx_ops, params.reference_for_reindex)
else:
resolve_indexing_ambiguity(dirs, reidx_ops)
ofs = open(params.lstout, "w")
for wd in dirs:
xas_full = os.path.join(wd, "XDS_ASCII_fullres.HKL")
if os.path.isfile(xas_full): # if _fullres.HKL exists, use it.
ofs.write("%s\n" % xas_full)
else:
ofs.write("%s\n" % os.path.join(wd, "XDS_ASCII.HKL"))
#ksb.debug_write_mtz()
#ksb.modify_xds_ascii_files()
quit()
arrays = []
for f in xac_files:
print "Reading", f
xac = XDS_ASCII(f, i_only=True)
xac.remove_rejected()
a = xac.i_obs().resolution_filter(d_min=3)
a = a.merge_equivalents(use_internal_variance=False).array()
arrays.append(a)
symm = arrays[0].crystal_symmetry()
cosets = reindex.reindexing_operators(symm, symm)
reidx_ops = cosets.combined_cb_ops()
reidx_ops.sort(key=lambda x: not x.is_identity_op())
print " Possible reindex operators:", map(lambda x: str(x.as_hkl()),
reidx_ops)
determined = set([
0,
])
old_ops = map(lambda x: 0, xrange(len(arrays)))
for ncycle in xrange(100): # max cycle
new_ops = map(lambda x: 0, xrange(len(arrays)))
for i in xrange(len(arrays)):
cc_list = []
a = arrays[i]
def find_reindex_ops(self):
symm = self.arrays[0].crystal_symmetry()
cosets = reindex.reindexing_operators(symm, symm, max_delta=self.max_delta)
reidx_ops = cosets.combined_cb_ops()
return reidx_ops
def rescale_with_specified_symm_worker(sym_wd_wdr,
topdir,
log_out,
reference_symm,
sgnum,
sgnum_laue,
prep_dials_files=False):
# XXX Unsafe if multiple processes run this function for the same target directory at the same time
sym, wd, wdr = sym_wd_wdr
out = StringIO()
print >> out, os.path.relpath(wd, topdir),
# Find appropriate data # XXX not works for DIALS data!!
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
log_out.write(out.getvalue())
log_out.flush()
return (wd, None)
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1,
10):
print >> out, " Already scaled with specified symmetry"
log_out.write(out.getvalue())
log_out.flush()
if wd != wdr: shutil.copy2(xac_file, wdr)
if prep_dials_files: prepare_dials_files(wd, out, moveto=wdr)
return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
os.path.join(wdr, os.path.basename(xac_file))))
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
log_out.write(out.getvalue())
log_out.flush()
return (wdr, None)
newcell = reference_symm.space_group().average_unit_cell(
xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT,
wdir=wd,
quiet=True)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f),
os.path.join(wdr, f + "_rescale"))
revert_files(xds_files.generated_by_CORRECT,
bk_prefix,
wdir=wd,
quiet=True)
new_xac = os.path.join(wdr, "XDS_ASCII.HKL_rescale")
if prep_dials_files:
prepare_dials_files(wd,
out,
space_group=reference_symm.space_group(),
reindex_op=cosets.combined_cb_ops()[0],
moveto=wdr)
ret = None
if os.path.isfile(new_xac):
ret = (XDS_ASCII(new_xac,
read_data=False).symm.unit_cell().parameters(),
new_xac)
print >> out, " OK:", ret[0]
else:
print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
return (wd, ret)
def check_cell(params, xs):
assert params.checkcell.check and params.sgnum > 0
xsref = crystal.symmetry(params.cell, params.sgnum)
cosets = reindex.reindexing_operators(xsref, xs,
params.checkcell.tol_length, params.checkcell.tol_angle)
return cosets
def reindex_with_specified_symm_worker(wd,
wdr,
topdir,
log_out,
reference_symm,
sgnum_laue,
prep_dials_files=False):
"""
wd: directory where XDS file exists
wdr: wd to return; a directory where transformed file should be saved.
If wd!=wdr, files in wd/ are unchanged during procedure. Multiprocessing is unsafe when wd==wdr.
"""
out = StringIO()
print >> out, "%s:" % os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL", "DIALS.HKL.org", "DIALS.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
log_out.write(out.getvalue())
log_out.flush()
return (wdr, None)
if xac_file.endswith(".org"): xac_file_out = xac_file[:-4]
else: xac_file_out = xac_file
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(
reference_symm.unit_cell(), 0.1,
10): # XXX Check unit cell consistency!!
print >> out, " Already scaled with specified symmetry"
log_out.write(out.getvalue())
log_out.flush()
if wd != wdr: shutil.copy2(xac_file, wdr)
if prep_dials_files and "DIALS.HKL" not in xac_file:
prepare_dials_files(wd, out, moveto=wdr)
return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
os.path.join(wdr, os.path.basename(xac_file))))
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
20) # XXX ISN'T THIS TOO LARGE?
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
xac.symm.show_summary(out, " ")
reference_symm.show_summary(out, " ")
log_out.write(out.getvalue())
log_out.flush()
return (wdr, None)
if wd == wdr:
dest = tempfile.mkdtemp(prefix="multiprep", dir=wd)
else:
dest = wdr
hklout = os.path.join(dest, os.path.basename(xac_file_out))
newcell = xac.write_reindexed(op=cosets.combined_cb_ops()[0],
space_group=reference_symm.space_group(),
hklout=hklout)
if "DIALS.HKL" in os.path.basename(xac_file):
outstr = 'output.experiments="%sreindexed_experiments.json" ' % os.path.join(
dest, "")
outstr += 'output.reflections="%sreindexed_reflections.pickle" ' % os.path.join(
dest, "")
for f in ("experiments.json", "indexed.pickle"):
if not os.path.isfile(os.path.join(os.path.dirname(xac_file), f)):
continue
util.call(
'dials.reindex %s change_of_basis_op=%s space_group="%s" %s' %
(f, cosets.combined_cb_ops()[0].as_abc(),
reference_symm.space_group_info(), outstr),
wdir=os.path.dirname(xac_file))
elif prep_dials_files:
prepare_dials_files(wd,
out,
space_group=reference_symm.space_group(),
reindex_op=cosets.combined_cb_ops()[0],
moveto=dest)
newcell_str = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, " Reindexed to transformed cell: %s with %s" % (
newcell_str, cosets.combined_cb_ops()[0].as_hkl())
log_out.write(out.getvalue())
log_out.flush()
if wd == wdr:
for f in glob.glob(os.path.join(dest, "*")):
f_in_wd = os.path.join(wd, os.path.basename(f))
if os.path.exists(f_in_wd) and not os.path.exists(f_in_wd +
".org"):
os.rename(f_in_wd, f_in_wd + ".org")
os.rename(f, f_in_wd)
shutil.rmtree(dest)
ret = (numpy.array(newcell.parameters()),
os.path.join(wd, os.path.basename(xac_file_out)))
else:
ret = (numpy.array(newcell.parameters()), hklout)
return (wdr, ret)
def xds_sequence(root, params):
print
print os.path.relpath(root, params.topdir)
init_lp = os.path.join(root, "INIT.LP")
xparm = os.path.join(root, "XPARM.XDS")
gxparm = os.path.join(root, "GXPARM.XDS")
defpix_lp = os.path.join(root, "DEFPIX.LP")
correct_lp = os.path.join(root, "CORRECT.LP")
integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
integrate_lp = os.path.join(root, "INTEGRATE.LP")
spot_xds = os.path.join(root, "SPOT.XDS")
xdsinp = os.path.join(root, "XDS.INP")
assert os.path.isfile(xdsinp)
if params.cell_prior.force: assert params.cell_prior.check
xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))
if params.cell_prior.sgnum > 0:
xs_prior = crystal.symmetry(params.cell_prior.cell,
params.cell_prior.sgnum)
else:
xs_prior = None
decilog = multi_out()
decilog.register("log",
open(os.path.join(root, "decision.log"), "a"),
atexit_send_to=None)
try:
print >> decilog, "xds_sequence started at %s in %s\n" % (
time.strftime("%Y-%m-%d %H:%M:%S"), root)
if not kamo_test_installation.tst_xds():
print >> decilog, "XDS is not installed or expired!!"
return
if params.show_progress:
decilog.register("stdout", sys.stdout)
if params.mode == "initial" and params.resume and os.path.isfile(
correct_lp):
print >> decilog, " Already processed."
return
if params.mode == "recycle" and not os.path.isfile(gxparm):
print >> decilog, "GXPARM.XDS not found. Cannot do recycle."
return
if params.fast_delphi and (params.nproc is None or params.nproc > 1):
delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
print >> decilog, " Setting delphi to ", delphi
modify_xdsinp(xdsinp, inp_params=[
("DELPHI", str(delphi)),
])
if params.nproc is not None and params.nproc > 1:
modify_xdsinp(xdsinp,
inp_params=[
("MAXIMUM_NUMBER_OF_PROCESSORS",
str(params.nproc)),
])
if params.mode == "initial":
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT")])
run_xds(wdir=root, show_progress=params.show_progress)
initlp = InitLp(init_lp)
first_bad = initlp.check_bad_first_frames()
if first_bad:
print >> decilog, " first frames look bad (too weak) exposure:", first_bad
new_data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
new_data_range[0] = first_bad[-1] + 1
print >> decilog, " changing DATA_RANGE= to", new_data_range
modify_xdsinp(xdsinp,
inp_params=[("JOB", "INIT"),
("DATA_RANGE",
"%d %d" % tuple(new_data_range))])
for f in xds_files.generated_by_INIT:
util.rotate_file(os.path.join(root, f), copy=False)
run_xds(wdir=root, show_progress=params.show_progress)
# Peak search
modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
run_xds(wdir=root, show_progress=params.show_progress)
if params.auto_frame_exclude_spot_based:
sx = idxreflp.SpotXds(spot_xds)
sx.set_xdsinp(xdsinp)
spots = filter(lambda x: 5 < x[-1] < 30,
sx.collected_spots()) # low-res (5 A)
frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1,
spots))
data_range = map(
int,
dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
# XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
h = numpy.histogram(frame_numbers,
bins=numpy.arange(data_range[0],
data_range[1] + 2,
step=1))
q14 = numpy.percentile(h[0], [25, 75])
iqr = q14[1] - q14[0]
cutoff = max(
h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5 # magic number
print >> decilog, "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (
iqr, q14, cutoff)
cut_frames = h[1][h[0] < cutoff]
keep_frames = h[1][h[0] >= cutoff]
print >> decilog, "DEBUG:: keep_frames=", keep_frames
print >> decilog, "DEBUG:: cut_frames=", cut_frames
if len(cut_frames) > 0:
cut_ranges = [
[cut_frames[0], cut_frames[0]],
]
for fn in cut_frames:
if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
else: cut_ranges.append([fn, fn])
# Edit XDS.INP
cut_inp_str = "".join(
map(
lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(
x), cut_ranges))
open(xdsinp, "a").write("\n" + cut_inp_str)
# Edit SPOT.XDS
shutil.copyfile(spot_xds, spot_xds + ".org")
sx.write(open(spot_xds, "w"),
frame_selection=set(keep_frames))
# Indexing
if params.cell_prior.method == "use_first":
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
else:
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, "" # TODO indexing stats like indexed percentage here.
if params.tryhard:
try_indexing_hard(root,
params.show_progress,
decilog,
known_sgnum=params.cell_prior.sgnum,
known_cell=params.cell_prior.cell,
tol_length=params.cell_prior.tol_length,
tol_angle=params.cell_prior.tol_angle)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
if params.cell_prior.sgnum > 0:
# Check anyway
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(
xs_prior, xsxds, params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is None:
if params.cell_prior.check:
print >> decilog, " Incompatible cell. Indexing failed."
return
else:
print >> decilog, " Warning: Incompatible cell."
elif params.cell_prior.method == "symm_constraint_only":
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Trying symmetry-constrained cell parameter:", cell
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
for f in xds_files.generated_by_IDXREF:
util.rotate_file(os.path.join(root, f),
copy=(f == "SPOT.XDS"))
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xparm):
print >> decilog, " Indexing failed."
return
# Check again
xsxds = XPARM(xparm).crystal_symmetry()
if not xsxds.unit_cell().is_similar_to(
xs_prior.unit_cell(), params.cell_prior.tol_length,
params.cell_prior.tol_angle):
print >> decilog, " Resulted in different cell. Indexing failed."
return
elif params.mode == "recycle":
print >> decilog, " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
"plot_integrate.log"):
util.rotate_file(os.path.join(root, f), copy=True)
shutil.copyfile(gxparm + ".1", xparm)
else:
raise "Unknown mode (%s)" % params.mode
# To Integration
modify_xdsinp(xdsinp,
inp_params=[("JOB", "DEFPIX INTEGRATE"),
("INCLUDE_RESOLUTION_RANGE", "50 0")])
run_xds(wdir=root, show_progress=params.show_progress)
if os.path.isfile(integrate_lp):
xds_plot_integrate.run(integrate_lp,
os.path.join(root, "plot_integrate.log"))
if not os.path.isfile(integrate_hkl):
print >> decilog, " Integration failed."
return
# Make _noscale.HKL if needed
if params.no_scaling:
bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
xparm_obj = XPARM(xparm)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", "50"),
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
xparm_obj.unit_cell))),
("SPACE_GROUP_NUMBER",
"%d" % xparm_obj.spacegroup),
])
print >> decilog, " running CORRECT without empirical scaling"
run_xds(wdir=root, show_progress=params.show_progress)
for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
ff = os.path.join(root, f)
if not os.path.isfile(ff): continue
if ff.endswith(".cbf"):
os.remove(ff)
else:
os.rename(ff, ff + "_noscale")
revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)
# Run pointless
pointless_integrate = {}
if params.use_pointless:
worker = Pointless()
pointless_integrate = worker.run_for_symm(
xdsin=integrate_hkl,
logout=os.path.join(root, "pointless_integrate.log"))
if "symm" in pointless_integrate:
symm = pointless_integrate["symm"]
print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(symm.space_group(),
xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell)])
else:
print >> decilog, " pointless failed."
flag_do_not_change_symm = False
if xs_prior and params.cell_prior.force:
modify_xdsinp(xdsinp,
inp_params=[("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
params.cell_prior.cell))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum)])
flag_do_not_change_symm = True
elif params.cell_prior.method == "correct_only":
xsxds = XPARM(xparm).crystal_symmetry()
cosets = reindex.reindexing_operators(xs_prior, xsxds,
params.cell_prior.tol_length,
params.cell_prior.tol_angle)
if cosets.double_cosets is not None:
cell = xsxds.unit_cell().change_basis(
cosets.combined_cb_ops()[0])
print >> decilog, " Using given symmetry in CORRECT with symmetry constraints:", cell
modify_xdsinp(xdsinp,
inp_params=[
("UNIT_CELL_CONSTANTS", " ".join(
map(lambda x: "%.3f" % x,
cell.parameters()))),
("SPACE_GROUP_NUMBER",
"%d" % params.cell_prior.sgnum),
])
flag_do_not_change_symm = True
else:
print >> decilog, " Tried to use given symmetry in CORRECT, but cell in integration is incompatible."
# Do Scaling
modify_xdsinp(xdsinp, inp_params=[
("JOB", "CORRECT"),
])
run_xds(wdir=root, show_progress=params.show_progress)
if not os.path.isfile(xac_hkl):
print >> decilog, " CORRECT failed."
return
if not os.path.isfile(gxparm):
print >> decilog, " Refinement in CORRECT failed."
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(xac_hkl, os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root, show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)
# Run pointless and (if result is different from INTEGRATE) re-scale.
if params.use_pointless:
worker = Pointless()
pointless_correct = worker.run_for_symm(
xdsin=xac_hkl,
logout=os.path.join(root, "pointless_correct.log"))
pointless_best_symm = None
if "symm" in pointless_correct:
symm = pointless_correct["symm"]
need_rescale = False
if pointless_integrate.get("symm"):
symm_by_integrate = pointless_integrate["symm"]
if not xtal.is_same_laue_symmetry(
symm_by_integrate.space_group(),
symm.space_group()):
print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
symm.space_group_info(),
symm_by_integrate.space_group_info())
prob_integrate = pointless_integrate.get(
"laue_prob", float("nan"))
prob_correct = pointless_correct.get(
"laue_prob", float("nan"))
print >> decilog, " Prob(%s |INTEGRATE), Prob(%s |CORRECT) = %.4f, %.4f." % (
symm_by_integrate.space_group_info(),
symm.space_group_info(), prob_integrate,
prob_correct)
if prob_correct > prob_integrate:
need_rescale = True
pointless_best_symm = symm
else:
pointless_best_symm = symm_by_integrate
else:
need_rescale = True
pointless_best_symm = symm
print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
)
if xs_prior:
if xtal.is_same_space_group_ignoring_enantiomorph(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which is consistent with given symmetry."
elif xtal.is_same_laue_symmetry(
symm.space_group(), xs_prior.space_group()):
print >> decilog, " which has consistent Laue symmetry with given symmetry."
else:
print >> decilog, " which is inconsistent with given symmetry."
if need_rescale and not flag_do_not_change_symm:
sgnum = symm.space_group_info().type().number()
cell = " ".join(
map(lambda x: "%.2f" % x,
symm.unit_cell().parameters()))
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d" % sgnum),
("UNIT_CELL_CONSTANTS", cell),
("INCLUDE_RESOLUTION_RANGE", "50 0")
])
run_xds(wdir=root, show_progress=params.show_progress)
ret = calc_merging_stats(xac_hkl)
if params.cut_resolution:
if ret is not None and ret[0] is not None:
d_min = ret[0]
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("INCLUDE_RESOLUTION_RANGE",
"50 %.2f" % d_min)
])
print >> decilog, " Re-scale at %.2f A" % d_min
os.rename(os.path.join(root, "CORRECT.LP"),
os.path.join(root, "CORRECT_fullres.LP"))
os.rename(
xac_hkl,
os.path.join(root, "XDS_ASCII_fullres.HKL"))
run_xds(wdir=root,
show_progress=params.show_progress)
print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
correct_lp, check_valid=True)
print >> decilog, " (Original files are saved as *_fullres.*)"
else:
print >> decilog, "error: Can't decide resolution."
for f in ("CORRECT_fullres.LP",
"XDS_ASCII_fullres.HKL"):
if os.path.isfile(os.path.join(root, f)):
print >> decilog, "removing", f
os.remove(os.path.join(root, f))
ISa = correctlp.get_ISa(correct_lp, check_valid=True)
if ISa >= last_ISa or last_ISa != last_ISa: # if improved or last_ISa is nan
print >> decilog, "ISa improved= %.2f" % ISa
else:
print >> decilog, "ISa got worse= %.2f" % ISa
if pointless_best_symm:
xac_symm = XDS_ASCII(xac_hkl, read_data=False).symm
if not xtal.is_same_space_group_ignoring_enantiomorph(
xac_symm.space_group(),
pointless_best_symm.space_group()):
if xtal.is_same_laue_symmetry(
xac_symm.space_group(),
pointless_best_symm.space_group()):
tmp = "same Laue symmetry"
else:
tmp = "different Laue symmetry"
print >> decilog, "WARNING: symmetry in scaling is different from Pointless result (%s)." % tmp
run_xdsstat(wdir=root)
print
if params.make_report: html_report.make_individual_report(root, root)
except:
print >> decilog, traceback.format_exc()
finally:
print >> decilog, "\nxds_sequence finished at %s" % time.strftime(
"%Y-%m-%d %H:%M:%S")
decilog.close()
def reindex_with_specified_symm(topdir, reference_symm, dirs, out):
print >> out
print >> out, "Re-index to specified symmetry:"
reference_symm.show_summary(out, " ")
print >> out
print >> out
cells = {} # cell and file
sgnum_laue = reference_symm.space_group(
).build_derived_reflection_intensity_group(False).type().number()
for wd in dirs:
print >> out, "%s:" % os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
continue
if xac_file.endswith(".org"):
xac_file_org, xac_file = xac_file, xac_file[:-4]
else:
xac_file_org = xac_file + ".org"
if not os.path.isfile(xac_file_org):
os.rename(xac_file, xac_file_org)
xac = XDS_ASCII(xac_file_org, read_data=False)
print >> out, "%s %s (%s)" % (
os.path.basename(xac_file), xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(),
0.1, 10):
print >> out, " Already scaled with specified symmetry"
os.rename(xac_file_org, xac_file) # rename back
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()),
xac_file)
continue
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
20)
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
xac.symm.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = xac.write_reindexed(op=cosets.combined_cb_ops()[0],
space_group=reference_symm.space_group(),
hklout=xac_file)
cells[wd] = (numpy.array(newcell.parameters()), xac_file)
newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, " Reindexed to transformed cell: %s with %s" % (
newcell, cosets.combined_cb_ops()[0].as_hkl())
return cells
def rescale_with_specified_symm(topdir, dirs, symms, out, sgnum=None, reference_symm=None):
assert (sgnum, reference_symm).count(None) == 1
if sgnum is not None:
sgnum_laue = sgtbx.space_group_info(sgnum).group().build_derived_reflection_intensity_group(False).type().number()
matches = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)
matched_cells = numpy.array(map(lambda x: x.unit_cell().parameters(), matches))
median_cell = map(lambda x: numpy.median(matched_cells[:,x]), xrange(6))
reference_symm = crystal.symmetry(median_cell, sgnum)
else:
sgnum = reference_symm.space_group_info().type().number()
sgnum_laue = reference_symm.space_group().build_derived_reflection_intensity_group(False).type().number()
print >>out
print >>out, "Re-scaling with specified symmetry:", reference_symm.space_group_info().symbol_and_number()
print >>out, " reference cell:", reference_symm.unit_cell()
print >>out
print >>out
cells = {} # cell and file
for sym, wd in zip(symms, dirs):
print >>out, os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
wd=wd)
if xac_file is None:
print >>out, "Can't find XDS_ASCII file in %s" % wd
continue
xac = XDS_ASCII(xac_file, read_data=False)
print >>out, "%s %s (%s)" % (os.path.basename(xac_file), xac.symm.space_group_info(),
",".join(map(lambda x: "%.2f"%x, xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1, 10):
print >>out, " Already scaled with specified symmetry"
cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()), xac_file)
continue
xdsinp = os.path.join(wd, "XDS.INP")
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)
if len(cosets.combined_cb_ops())==0:
print >>out, "Can't find operator:"
sym.show_summary(out, " ")
reference_symm.show_summary(out, " ")
continue
newcell = reference_symm.space_group().average_unit_cell(xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
newcell = " ".join(map(lambda x: "%.3f"%x, newcell.parameters()))
print >>out, "Scaling with transformed cell:", newcell
#for f in xds_files.generated_by_CORRECT:
# util.rotate_file(os.path.join(wd, f))
bk_prefix = make_backup(xds_files.generated_by_CORRECT, wdir=wd, quiet=True)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("SPACE_GROUP_NUMBER", "%d"%sgnum),
("UNIT_CELL_CONSTANTS", newcell),
("INCLUDE_RESOLUTION_RANGE", "50 0"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("MINIMUM_I/SIGMA", None), # use default
("REFINE(CORRECT)", None), # use default
])
run_xds(wd)
for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
if os.path.exists(os.path.join(wd, f)):
shutil.copyfile(os.path.join(wd, f), os.path.join(wd, f+"_rescale"))
revert_files(xds_files.generated_by_CORRECT, bk_prefix, wdir=wd, quiet=True)
new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
print "OK:", cells[wd][0]
else:
print >>out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
continue
return cells, reference_symm
