def find_unconnected_xds_files(xds_files, min_ios=3, d_min=None):
import networkx as nx
from yamtbx.dataproc.xds import xds_ascii
from yamtbx.dataproc.xds import integrate_hkl_as_flex
if len(xds_files) < 2:
return []
G = nx.Graph()
arrays = []
for f in xds_files:
if xds_ascii.is_xds_ascii(f):
a = xds_ascii.XDS_ASCII(f, i_only=True).i_obs().resolution_filter(d_min=d_min)
elif integrate_hkl_as_flex.is_integrate_hkl(f):
a = integrate_hkl_as_flex.reader(f, ["IOBS", "SIGMA"]).i_obs().resolution_filter(d_min=d_min)
else:
raise "Never reaches here"
a = a.select(a.sigmas() > 0)
a = a.merge_equivalents(use_internal_variance=False).array()
arrays.append(a.select(a.data() / a.sigmas() >= min_ios))
for i in xrange(len(arrays) - 1):
for j in xrange(i + 1, len(arrays)):
matchs = arrays[i].match_indices(other=arrays[j], assert_is_similar_symmetry=False)
if matchs.pairs().size() >= 10:
G.add_edge(i, j)
print "edge", i, j
ccomps = map(lambda x: x, nx.connected_components(G))
print "DEBUG:: Connected components=", ccomps
keep_idxes = ccomps[0]
remove_idxes = filter(lambda x: x not in keep_idxes, xrange(len(xds_files)))
return remove_idxes
def run(files, params):
print "filename",
for cut in params.cut_ios: print "cut_ios_%.2f" % cut,
print
for f in files:
is_xac = xds_ascii.is_xds_ascii(f)
i_obs = None
if is_xac:
xac = xds_ascii.XDS_ASCII(f, read_data=True, i_only=True)
xac.remove_rejected()
i_obs = xac.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max)
if params.fix_variance_model:
ao, bo = xac.variance_model
an, bn = params.variance_model
i_obs = i_obs.customized_copy(sigmas = flex.sqrt(flex.abs(an * (i_obs.sigmas()**2/ao + (bn-bo)*flex.pow2(i_obs.data())))))
else:
ihkl = integrate_hkl_as_flex.reader(f, read_columns=("IOBS","SIGMA"))
i_obs = ihkl.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max)
if params.fix_variance_model:
a, b = params.variance_model
i_obs = i_obs.customized_copy(sigmas = flex.sqrt(a * (i_obs.sigmas()**2 + b*flex.pow2(i_obs.data()))))
cutoffs = eval_resolution(i_obs, params.n_bins, params.cut_ios)
print "%s %s" % (f, " ".join(map(lambda x: "%.2f"%x, cutoffs)))
def import_integrated(integrate_hkl, min_ios=3):
reader = integrate_hkl_as_flex.reader(integrate_hkl, "IOBS,SIGMA,XCAL,YCAL,ZCAL,XOBS,YOBS,ZOBS,ISEG".split(","))
# reference: dials/command_line/import_xds.py
table = flex.reflection_table()
table["id"] = flex.int(len(reader.hkl), 0)
table["panel"] = flex.size_t(len(reader.hkl), 0) # only assuming single panel
table["miller_index"] = reader.hkl
table["xyzcal.px.value"] = flex.vec3_double(reader.data["XCAL"], reader.data["YCAL"], reader.data["ZCAL"])
table["xyzobs.px.value"] = flex.vec3_double(reader.data["XOBS"], reader.data["YOBS"], reader.data["ZOBS"])
table["intensity.cor.value"] = reader.data["IOBS"]
table["intensity.cor.sigma"] = reader.data["SIGMA"] # not valid name, just for making selection
table["flags"] = flex.size_t(len(table), table.flags.indexed | table.flags.strong)
table = table.select(table["intensity.cor.sigma"] > 0)
table = table.select(table["intensity.cor.value"]/table["intensity.cor.sigma"] >= min_ios)
table = table.select(table["xyzobs.px.value"].norms() > 0) # remove invalid xyzobs
# dummy
table["xyzobs.px.variance"] = flex.vec3_double(len(table), (1,1,1)) # TODO appropriate variance value
table["s1"] = flex.vec3_double(len(table), (0,0,0)) # will be updated by set_obs_s1()
del table["intensity.cor.value"]
del table["intensity.cor.sigma"]
return table
def find_unconnected_xds_files(xds_files, min_ios=3, d_min=None):
import networkx as nx
from yamtbx.dataproc.xds import xds_ascii
from yamtbx.dataproc.xds import integrate_hkl_as_flex
if len(xds_files) < 2:
return []
G = nx.Graph()
arrays = []
for f in xds_files:
if xds_ascii.is_xds_ascii(f):
a = xds_ascii.XDS_ASCII(
f, i_only=True).i_obs().resolution_filter(d_min=d_min)
elif integrate_hkl_as_flex.is_integrate_hkl(f):
a = integrate_hkl_as_flex.reader(
f, ["IOBS", "SIGMA"]).i_obs().resolution_filter(d_min=d_min)
else:
raise "Never reaches here"
a = a.select(a.sigmas() > 0)
a = a.merge_equivalents(use_internal_variance=False).array()
arrays.append(a.select(a.data() / a.sigmas() >= min_ios))
for i in xrange(len(arrays) - 1):
for j in xrange(i + 1, len(arrays)):
matchs = arrays[i].match_indices(other=arrays[j],
assert_is_similar_symmetry=False)
if matchs.pairs().size() >= 10:
G.add_edge(i, j)
print "edge", i, j
ccomps = map(lambda x: x, nx.connected_components(G))
print "DEBUG:: Connected components=", ccomps
keep_idxes = ccomps[0]
remove_idxes = filter(lambda x: x not in keep_idxes,
xrange(len(xds_files)))
return remove_idxes
def run(params, out):
print >> out, "Frames:", params.frames
backup_needed = xds_files.generated_by_DEFPIX + (
"XDS.INP",
"BKGINIT.cbf",
)
bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)
ret = {} # {frame: [matches, spots, predicted]}
try:
# run DEFPIX to limit resolution.
modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
[("JOB", "DEFPIX"),
("INCLUDE_RESOLUTION_RANGE", "50 %.2f" % params.d_min)])
p = subprocess.Popen("xds", cwd=params.xdsdir)
p.wait()
# copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
os.path.join(params.xdsdir, "BKGINIT.cbf"))
for frame in params.frames:
print >> out, "Frame %d" % frame
print >> out, "====================\n"
# search spots
if params.spotfinder == "xds":
spotxds = get_colspot_result(frame_ranges=[
[frame, frame],
],
wdir=params.xdsdir)
spots = map(lambda x: x[:2],
spotxds.collected_spots(with_resolution=False))
else:
raise "Sorry!"
# run INTEGRATE to collect predicted coords
integrate_results = xds_predict_mitai.run(
param_source=os.path.join(params.xdsdir, "XPARM.XDS"),
frame_num=frame,
wdir=params.xdsdir,
need_adx=False,
sigmar=params.sigmar,
sigmab=params.sigmab)
# read predicted coords
tmp = filter(lambda x: x.endswith(".HKL"), integrate_results)
if len(tmp) == 0:
print >> out, "Integration failed!"
ret[frame] = (0, len(spots), 0)
continue
integrate_hkl = tmp[0]
cols = integrate_hkl_as_flex.reader(integrate_hkl, [],
False).get_column_names()
i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
predicted = []
for l in open(integrate_hkl):
if l.startswith("!"): continue
sp = l.split()
predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))
# compare them
nmatch = calc_matches(
spots, predicted, params.distance_limit_in_px,
open(
os.path.join(params.xdsdir,
"matched_predicted_%.4d.adx" % frame), "w"))
#nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
# open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))
ret[frame] = (nmatch, len(spots), len(predicted))
finally:
revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)
print >> out
for frame in sorted(ret):
nmatch, nspots, npredicted = ret[frame]
print >> out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (
frame, nmatch, npredicted, 100. * float(nmatch) /
npredicted if npredicted > 0 else float("nan"))
print >> out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (
frame, nmatch, nspots,
100. * float(nmatch) / nspots if nspots > 0 else float("nan"))
print >> out
return ret
def run(params, log_out):
xa = XDS_ASCII(params.xds_ascii, log_out)
rejected_array = miller.array(miller_set=miller.set(crystal_symmetry=xa.symm,
indices=xa.indices,
anomalous_flag=False),
data=xa.sigma_iobs < 0)
xa_zd = miller.array(miller_set=miller.set(crystal_symmetry=xa.symm,
indices=xa.indices,
anomalous_flag=False),
data=xa.zd)
# Read ZCAL, not ZOBS, because ZOBS (and XOBS, YOBS) can be zero (in case unobserved).
integ_data = integrate_hkl_as_flex.reader(params.integrate_hkl, ["MAXC","ZCAL"]).arrays()
maxc_array, integ_zcal = integ_data["MAXC"], integ_data["ZCAL"]
assert integ_zcal.unit_cell().is_similar_to(xa_zd.unit_cell()) # two set of indices should be comparable.
overload_flags = maxc_array.customized_copy(data=maxc_array.data() == params.overload)
print "Overloaded observations in INTEGRATE.HKL:", overload_flags.data().count(True)
print "Rejected (sigma<0) observations in XDS_ASCII.HKL:", rejected_array.data().count(True)
# common_sets() does not work correctly for unmerged data!
rejected_zd = xa_zd.select(rejected_array.data())
#reject_indices = flex.bool([False for i in xrange(overload_flags.size())])
print "making indices..........."
import yamtbx_utils_ext
integ_zcal = integ_zcal.sort(by_value="packed_indices") # Must be sorted before C++ function below!!
reject_indices = yamtbx_utils_ext.make_selection_for_xds_unmerged(rejected_zd.indices(),
rejected_zd.data(),
integ_zcal.indices(),
integ_zcal.data(),
3.)
"""
# This loop is too slow!
for i in xrange(rejected_zd.size()):
sel = integ_zcal.indices() == rejected_zd.indices()[i]
sel &= (integ_zcal.data() - rejected_zd.data()[i]) < 3
reject_indices.set_selected(sel, True)
print i, rejected_zd.size(), sel.count(True)
"""
"""
# This loop is also too slow!
for j in xrange(integ_zcal.size()): # j: INTEGRATE.HKL
if rejected_zd.indices()[i] != integ_zcal.indices()[j]:
continue
if abs(rejected_zd.data()[i] - integ_zcal.data()[j]) < 3: # within 3 frames.. OK?
reject_indices[j] = True
"""
print "Found rejected observations in INTEGRATE.HKL:", reject_indices.count(True)
overload_flags.data().set_selected(reject_indices, False) # Set 'Un-overloaded'
print "Remained overloaded observations:", overload_flags.data().count(True)
overload_flags_partial = overload_flags.map_to_asu().merge_equivalents(incompatible_flags_replacement=True).array()
overload_flags_all = overload_flags.map_to_asu().merge_equivalents(incompatible_flags_replacement=False).array()
mtz_object = iotbx.mtz.object(params.hklin). \
add_crystal("crystal", "project", overload_flags_all.unit_cell()). \
add_dataset(name="dataset", wavelength=0). \
add_miller_array(miller_array=overload_flags_all, column_root_label="SATURATED_ALL"). \
add_miller_array(miller_array=overload_flags_partial, column_root_label="SATURATED_PART"). \
mtz_object()
mtz_object.write(file_name=params.hklout)
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in+ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir): os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >>decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >>decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True, anomalous=params.anomalous,
spot_range=None, minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx, orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f"%params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f"%params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d"%params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f"%params.sepmin),
("CLUSTER_RADIUS", "%.2f"%params.cluster_radius),
("INDEX_ERROR", "%.4f"%params.index_error),
("INDEX_MAGNITUDE", "%d"%params.index_magnitude),
("INDEX_QUALITY", "%.2f"%params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
("VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels))
])
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >>decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f"%x, params.cell))),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >>decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min)),
])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell)))]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum)] + tmp)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f+".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(f, read_columns=["IOBS","SIGMA","XCAL","YCAL","RLP","PEAK","MAXC"])
pickle.dump(inhkl, open(f+".pkl", "w"), -1)
except ProcFailed, e:
print >>decilog, "Processing failed: %s" % e.message
def run(params, out):
print >>out, "Frames:", params.frames
backup_needed = xds_files.generated_by_DEFPIX + ("XDS.INP","BKGINIT.cbf",)
bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)
ret = {} # {frame: [matches, spots, predicted]}
try:
# run DEFPIX to limit resolution.
modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
[("JOB", "DEFPIX"), ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%params.d_min)])
p = subprocess.Popen("xds", cwd=params.xdsdir)
p.wait()
# copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
os.path.join(params.xdsdir, "BKGINIT.cbf"))
for frame in params.frames:
print >>out, "Frame %d" % frame
print >>out, "====================\n"
# search spots
if params.spotfinder == "xds":
spotxds = get_colspot_result(frame_ranges=[[frame, frame],], wdir=params.xdsdir)
spots = map(lambda x: x[:2], spotxds.collected_spots(with_resolution=False))
else:
raise "Sorry!"
# run INTEGRATE to collect predicted coords
integrate_results = xds_predict_mitai.run(param_source=os.path.join(params.xdsdir, "XPARM.XDS"),
frame_num=frame,
wdir=params.xdsdir, need_adx=False,
sigmar=params.sigmar, sigmab=params.sigmab)
# read predicted coords
tmp = filter(lambda x:x.endswith(".HKL"), integrate_results)
if len(tmp) == 0:
print >>out, "Integration failed!"
ret[frame] = (0, len(spots), 0)
continue
integrate_hkl = tmp[0]
cols = integrate_hkl_as_flex.reader(integrate_hkl, [], False).get_column_names()
i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
predicted = []
for l in open(integrate_hkl):
if l.startswith("!"): continue
sp = l.split()
predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))
# compare them
nmatch = calc_matches(spots, predicted, params.distance_limit_in_px,
open(os.path.join(params.xdsdir, "matched_predicted_%.4d.adx"%frame), "w"))
#nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
# open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))
ret[frame] = (nmatch, len(spots), len(predicted))
finally:
revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)
print >>out
for frame in sorted(ret):
nmatch, nspots, npredicted = ret[frame]
print >>out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (frame, nmatch, npredicted,
100.*float(nmatch)/npredicted if npredicted>0 else float("nan"))
print >>out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (frame, nmatch, nspots,
100.*float(nmatch)/nspots if nspots>0 else float("nan"))
print >>out
return ret
def run(params, log_out):
xa = XDS_ASCII(params.xds_ascii, log_out)
rejected_array = miller.array(miller_set=miller.set(
crystal_symmetry=xa.symm, indices=xa.indices, anomalous_flag=False),
data=xa.sigma_iobs < 0)
xa_zd = miller.array(miller_set=miller.set(crystal_symmetry=xa.symm,
indices=xa.indices,
anomalous_flag=False),
data=xa.zd)
# Read ZCAL, not ZOBS, because ZOBS (and XOBS, YOBS) can be zero (in case unobserved).
integ_data = integrate_hkl_as_flex.reader(params.integrate_hkl,
["MAXC", "ZCAL"]).arrays()
maxc_array, integ_zcal = integ_data["MAXC"], integ_data["ZCAL"]
assert integ_zcal.unit_cell().is_similar_to(
xa_zd.unit_cell()) # two set of indices should be comparable.
overload_flags = maxc_array.customized_copy(
data=maxc_array.data() == params.overload)
print "Overloaded observations in INTEGRATE.HKL:", overload_flags.data(
).count(True)
print "Rejected (sigma<0) observations in XDS_ASCII.HKL:", rejected_array.data(
).count(True)
# common_sets() does not work correctly for unmerged data!
rejected_zd = xa_zd.select(rejected_array.data())
#reject_indices = flex.bool([False for i in xrange(overload_flags.size())])
print "making indices..........."
import yamtbx_utils_ext
integ_zcal = integ_zcal.sort(
by_value="packed_indices"
) # Must be sorted before C++ function below!!
reject_indices = yamtbx_utils_ext.make_selection_for_xds_unmerged(
rejected_zd.indices(), rejected_zd.data(), integ_zcal.indices(),
integ_zcal.data(), 3.)
"""
# This loop is too slow!
for i in xrange(rejected_zd.size()):
sel = integ_zcal.indices() == rejected_zd.indices()[i]
sel &= (integ_zcal.data() - rejected_zd.data()[i]) < 3
reject_indices.set_selected(sel, True)
print i, rejected_zd.size(), sel.count(True)
"""
"""
# This loop is also too slow!
for j in xrange(integ_zcal.size()): # j: INTEGRATE.HKL
if rejected_zd.indices()[i] != integ_zcal.indices()[j]:
continue
if abs(rejected_zd.data()[i] - integ_zcal.data()[j]) < 3: # within 3 frames.. OK?
reject_indices[j] = True
"""
print "Found rejected observations in INTEGRATE.HKL:", reject_indices.count(
True)
overload_flags.data().set_selected(reject_indices,
False) # Set 'Un-overloaded'
print "Remained overloaded observations:", overload_flags.data().count(
True)
overload_flags_partial = overload_flags.map_to_asu().merge_equivalents(
incompatible_flags_replacement=True).array()
overload_flags_all = overload_flags.map_to_asu().merge_equivalents(
incompatible_flags_replacement=False).array()
mtz_object = iotbx.mtz.object(params.hklin). \
add_crystal("crystal", "project", overload_flags_all.unit_cell()). \
add_dataset(name="dataset", wavelength=0). \
add_miller_array(miller_array=overload_flags_all, column_root_label="SATURATED_ALL"). \
add_miller_array(miller_array=overload_flags_partial, column_root_label="SATURATED_PART"). \
mtz_object()
mtz_object.write(file_name=params.hklout)
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in + ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >> decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >> decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(
img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True,
anomalous=params.anomalous,
spot_range=None,
minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx,
orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance,
)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f" % params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f" % params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d" % params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f" % params.sepmin),
("CLUSTER_RADIUS", "%.2f" % params.cluster_radius),
("INDEX_ERROR", "%.4f" % params.index_error),
("INDEX_MAGNITUDE", "%d" % params.index_magnitude),
("INDEX_QUALITY", "%.2f" % params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
(
"VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS",
"%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels),
),
],
)
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
flag_try_cell_hint = True
if flag_try_cell_hint:
print >> decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
],
)
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0")])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >> decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
],
)
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min))])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE")])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [
("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [
("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(
xdsinp,
inp_params=[
("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
]
+ tmp,
)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >> decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle:
x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f + ".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >> decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(
f, read_columns=["IOBS", "SIGMA", "XCAL", "YCAL", "RLP", "PEAK", "MAXC"]
)
pickle.dump(inhkl, open(f + ".pkl", "w"), -1)
except ProcFailed, e:
print >> decilog, "Processing failed: %s" % e.message
