Пример #1
0
 def get_map_stats_for_atoms (self, atoms) :
   from cctbx import maptbx
   from scitbx.array_family import flex
   sites_cart = flex.vec3_double()
   sites_cart_nonH = flex.vec3_double()
   values_2fofc = flex.double()
   values_fofc = flex.double()
   for atom in atoms :
     sites_cart.append(atom.xyz)
     if (not atom.element.strip() in ["H","D"]) : #XXX trap: neutrons?
       sites_cart_nonH.append(atom.xyz)
       site_frac = self.unit_cell.fractionalize(atom.xyz)
       values_2fofc.append(self.f_map.eight_point_interpolation(site_frac))
       values_fofc.append(self.diff_map.eight_point_interpolation(site_frac))
   if (len(sites_cart_nonH) == 0) :
     return None
   sel = maptbx.grid_indices_around_sites(
     unit_cell=self.unit_cell,
     fft_n_real=self.f_map.focus(),
     fft_m_real=self.f_map.all(),
     sites_cart=sites_cart,
     site_radii=get_atom_radii(atoms, self.atom_radius))
   f_map_sel = self.f_map.select(sel)
   model_map_sel = self.model_map.select(sel)
   diff_map_sel = self.diff_map.select(sel)
   cc = flex.linear_correlation(x=f_map_sel, y=model_map_sel).coefficient()
   return group_args(cc=cc,
     mean_2fofc=flex.mean(values_2fofc),
     mean_fofc=flex.mean(values_fofc))
Пример #2
0
 def show(self):
   b = self.bss_result
   print >> self.log, "  Statistics in resolution bins:"
   #assert k_mask.size() == len(self.bin_selections)
   fmt="  %7.5f %6.2f -%6.2f %5.1f %5d %-6s %-6s %-6s  %6.3f %6.3f %8.2f %6.4f"
   f_model = self.core.f_model.data()
   print >> self.log, "  s^2      Resolution    Compl Nrefl k_mask                 k_iso  k_ani <Fobs>   R"
   print >> self.log, "                (A)        (%)       orig   smooth average"
   k_mask_bin_orig_   = str(None)
   k_mask_bin_smooth_ = str(None)
   k_mask_bin_approx_ = str(None)
   for i_sel, cas in enumerate(self.cores_and_selections):
     selection, core, selection_use, sel_work = cas
     sel = sel_work
     ss_ = self.ss_bin_values[i_sel][2]
     if(b is not None and self.bss_result.k_mask_bin_orig is not None):
       k_mask_bin_orig_ = "%6.4f"%self.bss_result.k_mask_bin_orig[i_sel]
     if(b is not None and self.bss_result.k_mask_bin_smooth is not None):
       k_mask_bin_smooth_ = "%6.4f"%self.bss_result.k_mask_bin_smooth[i_sel]
     k_mask_bin_averaged_ = "%6.4f"%flex.mean(self.core.k_mask().select(sel))
     d_             = self.d_spacings.data().select(sel)
     d_min_         = flex.min(d_)
     d_max_         = flex.max(d_)
     n_ref_         = d_.size()
     f_obs_         = self.f_obs.select(sel)
     f_obs_mean_    = flex.mean(f_obs_.data())
     k_isotropic_   = flex.mean(self.core.k_isotropic.select(sel))
     k_anisotropic_ = flex.mean(self.core.k_anisotropic.select(sel))
     cmpl_          = f_obs_.completeness(d_max=d_max_)*100.
     r_             = bulk_solvent.r_factor(f_obs_.data(),f_model.select(sel),1)
     print >> self.log, fmt%(ss_, d_max_, d_min_, cmpl_, n_ref_,
       k_mask_bin_orig_, k_mask_bin_smooth_,k_mask_bin_averaged_,
       k_isotropic_, k_anisotropic_, f_obs_mean_, r_)
  def optimize(self):
    # initialise the population please
    self.make_random_population()
    # score the population please
    self.score_population()
    converged = False
    monitor_score = flex.min( self.scores )
    self.count = 0
    while not converged:
      self.evolve()
      location = flex.min_index( self.scores )
      if self.show_progress:
        if self.count%self.show_progress_nth_cycle==0:
          # make here a call to a custom print_status function in the evaluator function
          # the function signature should be (min_target, mean_target, best vector)
          self.evaluator.print_status(
            flex.min(self.scores),
            flex.mean(self.scores),
            self.population[ flex.min_index( self.scores ) ],
            self.count)

      self.count += 1
      if self.count%self.monitor_cycle==0:
        if (monitor_score-flex.min(self.scores) ) < self.eps:
          converged = True
        else:
         monitor_score = flex.min(self.scores)
      rd = (flex.mean(self.scores) - flex.min(self.scores) )
      rd = rd*rd/(flex.min(self.scores)*flex.min(self.scores) + self.eps )
      if ( rd < self.eps ):
        converged = True


      if self.count>=self.max_iter:
        converged =True
Пример #4
0
def log_frame(experiments, reflections, params, run, n_strong, timestamp = None, two_theta_low = None, two_theta_high = None):
  app = dxtbx_xfel_db_application(params)
  db_run = app.get_run(run_number=run)
  if params.input.trial is None:
    db_trial = app.get_trial(trial_id = params.input.trial_id)
    params.input.trial = db_trial.trial
  else:
    db_trial = app.get_trial(trial_number = params.input.trial)

  if params.input.rungroup is None:
    db_event = app.create_event(timestamp = timestamp, run_id = db_run.id, trial_id = db_trial.id, n_strong = n_strong,
      two_theta_low = two_theta_low, two_theta_high = two_theta_high)
  else:
    db_event = app.create_event(timestamp = timestamp, run_id = db_run.id, trial_id = db_trial.id, rungroup_id = params.input.rungroup, n_strong = n_strong,
      two_theta_low = two_theta_low, two_theta_high = two_theta_high)

  if experiments is not None:
    assert len(experiments) == 1
    db_experiment = app.create_experiment(experiments[0])
    app.link_imageset_frame(db_experiment.imageset, db_event)

    d = experiments[0].crystal.get_unit_cell().d(reflections['miller_index'])
    for db_bin in db_experiment.crystal.cell.bins: # will be [] if there are no isoforms
      sel = (d <= float(db_bin.d_max)) & (d > float(db_bin.d_min))
      sel &= reflections['intensity.sum.value'] > 0
      refls = reflections.select(sel)
      n_refls = len(refls)
      Cell_Bin(app,
               count = n_refls,
               bin_id = db_bin.id,
               crystal_id = db_experiment.crystal.id,
               avg_intensity = flex.mean(refls['intensity.sum.value']) if n_refls > 0 else None,
               avg_sigma = flex.mean(flex.sqrt(refls['intensity.sum.variance'])) if n_refls > 0 else None,
               avg_i_sigi = flex.mean(refls['intensity.sum.value'] /
                                      flex.sqrt(refls['intensity.sum.variance'])) if n_refls > 0 else None)
Пример #5
0
 def is_converged(self, rho_trial):
   result = False
   r = r_factor(self.f, self.f_mem, use_scale=False)
   if(r < self.convergence_r_threshold):
     if(self.xray_structure is None):
       self.r_factors.append(r)
       size = self.r_factors.size()
       if(size>=3):
         tmp = flex.mean(self.r_factors[size-3:])
         if(tmp <= r or r <= self.convergence_at_r_factor): result = True
     else:
       f_mem = self.full_set.structure_factors_from_map(
         map            = rho_trial,
         use_scale      = False,
         anomalous_flag = False,
         use_sg         = False)
       self.cc = f_mem.map_correlation(other = self.f_calc)
       self.cc_to_answer.append(self.cc)
       def max_change_so_far(x):
         result = flex.double()
         if(self.cc_to_answer.size()):
           for i in xrange(self.cc_to_answer.size()):
             if(i>0):
               result.append(self.cc_to_answer[i]-self.cc_to_answer[i-1])
         return flex.max(result)
       size = self.cc_to_answer.size()
       if(size>=3):
         mcsf = max_change_so_far(x = self.cc_to_answer)
         tmp = flex.mean(self.cc_to_answer[size-3:])
         if(tmp >= self.cc-1.e-6 or mcsf/5 >= self.cc-tmp):
           result = True
   else: self.cc=None
   return result
def exercise():
  """
  Exercise refine "easy" with DNA/RNA.
  """
  pi_good = get_pdb_inputs(pdb_str=pdb_str_answer, restraints=False)
  map_data = get_map(xrs=pi_good.xrs)
  xrs_good = pi_good.xrs.deep_copy_scatterers()
  pi_good.ph.write_pdb_file(file_name="answer.pdb",
    crystal_symmetry=xrs_good.crystal_symmetry())
  #
  pi_poor = get_pdb_inputs(pdb_str=pdb_str_poor, restraints=True)
  pi_poor.ph.write_pdb_file(file_name="poor.pdb")
  xrs_poor = pi_poor.xrs.deep_copy_scatterers()
  #
  d = xrs_good.distances(other=xrs_poor)
  print d.min_max_mean().as_tuple()
  assert flex.max(d)>2
  assert flex.mean(d)>0.7
  #
  xrs_refined = xrs_poor
  for i in xrange(3):
    ero = individual_sites.easy(
      map_data                    = map_data,
      xray_structure              = xrs_refined,
      pdb_hierarchy               = pi_poor.ph,
      geometry_restraints_manager = pi_poor.grm)
    xrs_refined = ero.xray_structure
  # comapre
  d = xrs_good.distances(other=xrs_refined)
  print d.min_max_mean().as_tuple()
  assert flex.max(d)<0.15
  assert flex.mean(d)<0.03
  ero.pdb_hierarchy.write_pdb_file(file_name="refined.pdb",
    crystal_symmetry=xrs_good.crystal_symmetry())
def exercise_gauss():
  data = rt.normal_variate(mu=0,sigma=1,N=1000000)
  mu1 = flex.mean(data)
  mu2 = flex.mean(data*data)
  mu3 = flex.mean(data*data*data)
  assert approx_equal(mu1,0,eps=0.02)
  assert approx_equal(mu2,1,eps=0.02)
  assert approx_equal(mu3,0,eps=0.02)
Пример #8
0
 def __init__ (self,
     ligand,
     pdb_hierarchy,
     xray_structure,
     two_fofc_map,
     fofc_map,
     fmodel_map,
     reference_ligands=None,
     two_fofc_map_cutoff=1.5,
     fofc_map_cutoff=-3.0) :
   from mmtbx import real_space_correlation
   from cctbx import adptbx
   from scitbx.array_family import flex
   atom_selection = ligand.atoms().extract_i_seq()
   assert (len(atom_selection) == 1) or (not atom_selection.all_eq(0))
   manager = real_space_correlation.selection_map_statistics_manager(
     atom_selection=atom_selection,
     xray_structure=xray_structure,
     fft_m_real=two_fofc_map.all(),
     fft_n_real=two_fofc_map.focus(),
     exclude_hydrogens=True)
   stats_two_fofc = manager.analyze_map(
     map=two_fofc_map,
     model_map=fmodel_map,
     min=1.5)
   stats_fofc = manager.analyze_map(
     map=fofc_map,
     model_map=fmodel_map,
     min=-3.0)
   self.atom_selection = manager.atom_selection # XXX non-hydrogens only!
   sites_cart = xray_structure.sites_cart().select(self.atom_selection)
   self.xyz_center = sites_cart.mean()
   self.id_str = ligand.id_str()
   self.cc = stats_two_fofc.cc
   self.two_fofc_min = stats_two_fofc.min
   self.two_fofc_max = stats_two_fofc.max
   self.two_fofc_mean = stats_two_fofc.mean
   self.fofc_min = stats_fofc.min
   self.fofc_max = stats_fofc.max
   self.fofc_mean = stats_fofc.mean
   self.n_below_two_fofc_cutoff = stats_two_fofc.n_below_min
   self.n_below_fofc_cutoff = stats_fofc.n_below_min
   u_iso = xray_structure.extract_u_iso_or_u_equiv().select(
     self.atom_selection)
   u_iso_mean = flex.mean(u_iso)
   self.b_iso_mean = adptbx.u_as_b(u_iso_mean)
   occ = xray_structure.scatterers().extract_occupancies().select(
     self.atom_selection)
   self.occupancy_mean = flex.mean(occ)
   self.rmsds = self.pbss = None
   if (reference_ligands is not None) and (len(reference_ligands) > 0) :
     self.rmsds, self.pbss = compare_ligands_impl(ligand=ligand,
       reference_ligands=reference_ligands,
       max_distance_between_centers_of_mass=8.0,
       raise_sorry_if_no_matching_atoms=False,
       verbose=False,
       quiet=True)
Пример #9
0
def exercise_variate_generators():
  from scitbx.random \
       import variate, normal_distribution, bernoulli_distribution, \
              gamma_distribution, poisson_distribution
  for i in xrange(10):
    scitbx.random.set_random_seed(0)
    g = variate(normal_distribution())
    assert approx_equal(g(), -1.2780081289048213)
    assert approx_equal(g(10),
      (-0.40474189234755492, -0.41845505596083288,
       -1.8825790263067721, -1.5779112018107659,
       -1.1888174422378859, -1.8619619179878537,
       -0.53946818661388318, -1.2400941724410812,
       0.64511959841907285, -0.59934120033270688))

  stat = basic_statistics(flex.double(itertools.islice(g, 1000000)))
  assert approx_equal(stat.mean,            0, eps=0.005)
  assert approx_equal(stat.biased_variance, 1, eps=0.005)
  assert approx_equal(stat.skew,            0, eps=0.005)
  assert approx_equal(stat.kurtosis,        3, eps=0.005)

  bernoulli_seq = variate(bernoulli_distribution(0.1))
  for b in itertools.islice(bernoulli_seq, 10):
    assert b in (True, False)
  bernoulli_sample = flex.bool(itertools.islice(bernoulli_seq, 10000))
  assert approx_equal(
    bernoulli_sample.count(True)/len(bernoulli_sample),
    0.1,
    eps = 0.01)

  scitbx.random.set_random_seed(0)
  g = variate(gamma_distribution())
  assert approx_equal(g(), 0.79587450456577546)
  assert approx_equal(g(2), (0.89856038848394115, 1.2559307580473893))
  stat = basic_statistics(flex.double(itertools.islice(g, 1000000)))
  assert approx_equal(stat.mean,            1, eps=0.005)
  assert approx_equal(stat.skew,            2, eps=0.005)
  assert approx_equal(stat.biased_variance, 1, eps=0.005)
  scitbx.random.set_random_seed(0)
  g = variate(gamma_distribution(alpha=2, beta=3))
  assert approx_equal(g(), 16.670850592722729)
  assert approx_equal(g(2), (10.03662877519449, 3.9357158398972873))
  stat = basic_statistics(flex.double(itertools.islice(g, 1000000)))
  assert approx_equal(stat.mean,            6, eps=0.005)
  assert approx_equal(stat.skew,            2/math.sqrt(2), eps=0.05)
  assert approx_equal(stat.biased_variance, 18, eps=0.05)

  mean = 10.0
  pv = variate(poisson_distribution(mean))
  draws = pv(1000000).as_double()
  m = flex.mean(draws)
  v = flex.mean(draws*draws) - m*m
  assert approx_equal(m,mean,eps=0.05)
  assert approx_equal(v,mean,eps=0.05)
def vectors(handle,all):
  from scitbx.array_family import flex
  x = flex.double()
  y = flex.double()
  xpred = flex.double()
  ypred = flex.double()
  for line in handle.readlines():
    if line.find("CV ")!=0: continue
    tokens = line.split()
    obscen = matrix.col((float(tokens[2]),float(tokens[3])))
    refcen = matrix.col((float(tokens[5]),float(tokens[6])))
    obsspo = matrix.col((float(tokens[8]),float(tokens[9])))
    predspo = matrix.col((float(tokens[11]),float(tokens[12])))
    xpred.append(predspo[0])
    ypred.append(predspo[1])
    prediction = predspo-refcen
    observation = obsspo-obscen
    cv = prediction-observation
    x.append(cv[0])
    y.append(cv[1])

  if all:
    print "Plotting all %d spots one graph."%len(x)
    from matplotlib import pyplot as plt
    plt.plot(x,y,"r.")
    plt.show()
    return

  print len(x),len(y)
  #from spotfinder.applications.xfel.cxi_run3 import get_initial_cxi_scope
  #print "ONLY FOR RUN 3!!!"
  #params = get_initial_cxi_scope()
  #tiling = params.distl.detector_tiling

  print "ONLY FOR RUN 4!!!"
  tiling = [518, 439, 712, 624, 715, 439, 909, 624, 519, 652, 713, 837, 716, 652, 910, 837, 510, 19, 695, 213, 510, 216, 695, 410, 721, 19, 906, 213, 721, 216, 906, 410, 87, 233, 281, 418, 284, 233, 478, 418, 88, 20, 282, 205, 285, 20, 479, 205, 108, 447, 293, 641, 108, 644, 293, 838, 321, 445, 506, 639, 321, 642, 506, 836, 437, 853, 622, 1047, 437, 1050, 622, 1244, 649, 853, 834, 1047, 649, 1050, 834, 1244, 19, 1069, 213, 1254, 216, 1069, 410, 1254, 18, 856, 212, 1041, 215, 856, 409, 1041, 230, 1282, 415, 1476, 230, 1479, 415, 1673, 16, 1282, 201, 1476, 16, 1479, 201, 1673, 442, 1469, 636, 1654, 639, 1469, 833, 1654, 443, 1257, 637, 1442, 640, 1257, 834, 1442, 852, 1137, 1046, 1322, 1049, 1137, 1243, 1322, 852, 925, 1046, 1110, 1049, 925, 1243, 1110, 1067, 1350, 1252, 1544, 1067, 1547, 1252, 1741, 854, 1352, 1039, 1546, 854, 1549, 1039, 1743, 1280, 1342, 1474, 1527, 1477, 1342, 1671, 1527, 1282, 1554, 1476, 1739, 1479, 1554, 1673, 1739, 1467, 924, 1652, 1118, 1467, 1121, 1652, 1315, 1255, 925, 1440, 1119, 1255, 1122, 1440, 1316, 1142, 521, 1327, 715, 1142, 718, 1327, 912, 930, 521, 1115, 715, 930, 718, 1115, 912, 1359, 514, 1553, 699, 1556, 514, 1750, 699, 1358, 727, 1552, 912, 1555, 727, 1749, 912, 1353, 92, 1538, 286, 1353, 289, 1538, 483, 1565, 91, 1750, 285, 1565, 288, 1750, 482, 932, 111, 1126, 296, 1129, 111, 1323, 296, 931, 323, 1125, 508, 1128, 323, 1322, 508]

  for itile in xrange(len(tiling)//4):
    print "tile",itile,
    print "(%4d %4d)-(%4d %4d)"%tuple(tiling[4*itile:4*itile+4]),
    selection = flex.bool()
    for i in xrange(len(x)):
      selection.append(
         tiling[4*itile+0]<xpred[i]<tiling[4*itile+2] and
         tiling[4*itile+1]<ypred[i]<tiling[4*itile+3]
     )
    print "in selection of %d/%d"%(selection.count(True),len(selection))
    if selection.count(True)<10:continue
    from matplotlib import pyplot as plt
    plt.plot(x.select(selection),y.select(selection),"r.")
    plt.plot([flex.mean(x.select(selection))],[flex.mean(y.select(selection))],"go")
    print "Delta x=%.1f"%flex.mean(x.select(selection)),"Delta y=%.1f"%flex.mean(y.select(selection))
    plt.show()
Пример #11
0
  def plot_uc_3Dplot(self, info):
    assert self.interactive

    import numpy as np
    from mpl_toolkits.mplot3d import Axes3D # import dependency

    fig = self.plt.figure(figsize=(12, 10))
    # Extract uc dimensions from info list
    a = flex.double([i['a'] for i in info])
    b = flex.double([i['b'] for i in info])
    c = flex.double([i['c'] for i in info])
    alpha = flex.double([i['alpha'] for i in info])
    beta = flex.double([i['beta'] for i in info])
    gamma = flex.double([i['gamma'] for i in info])
    n_total = len(a)

    accepted = flex.bool(n_total, True)
    for d in [a, b, c, alpha, beta, gamma]:
      outliers = self.reject_outliers(d)
      accepted &= ~outliers

    a = a.select(accepted)
    b = b.select(accepted)
    c = c.select(accepted)

    AA = "a-edge (%.2f +/- %.2f $\AA$)" % (flex.mean(a),
                                        flex.mean_and_variance(a).unweighted_sample_standard_deviation())
    BB = "b-edge (%.2f +/- %.2f $\AA$)" % (flex.mean(b),
                                        flex.mean_and_variance(b).unweighted_sample_standard_deviation())
    CC = "c-edge (%.2f +/- %.2f $\AA$)" % (flex.mean(c),
                                        flex.mean_and_variance(c).unweighted_sample_standard_deviation())


    subset = min(len(a),1000)

    flex.set_random_seed(123)
    rnd_sel = flex.random_double(len(a))<(subset/n_total)

    a = a.select(rnd_sel)
    b = b.select(rnd_sel)
    c = c.select(rnd_sel)

    fig.suptitle('{} randomly selected cells out of total {} images'
                 ''.format(len(a),n_total), fontsize=18)

    ax = fig.add_subplot(111, projection='3d')

    for ia in xrange(len(a)):
      ax.scatter(a[ia],b[ia],c[ia],c='r',marker='+')

    ax.set_xlabel(AA)
    ax.set_ylabel(BB)
    ax.set_zlabel(CC)
  def explore(self):
     if(self.count == 0):
       self.T=(flex.mean(flex.pow2(self.simplexValue - flex.mean(self.simplexValue) )))**0.5 * 10.0
       self.min_T = self.T /self.T_ratio
     elif(self.count%self.Nstep == 0):
       self.T = self.T*self.coolfactor

     for kk in range(1,self.dimension+1):
       self.FindCentroidPt(self.dimension+1-kk)
       self.FindReflectionPt(kk)
     self.sort()

     return # end of this explore step
Пример #13
0
def exercise_poisson():
  import random
  random.seed(0) # failure rate is fairly high without fixed seed
  a = rt.poisson_variate(100000,1).as_double()
  m = flex.mean(a)
  v = flex.mean(a*a)-m*m
  assert approx_equal(m,1.0,eps=0.05)
  assert approx_equal(v,1.0,eps=0.05)

  a = rt.poisson_variate(100000,10).as_double()
  m = flex.mean(a)
  v = flex.mean(a*a)-m*m
  assert approx_equal(m,10.0,eps=0.05)
  assert approx_equal(v,10.0,eps=0.05)
def exercise_ellipsoidal_truncation(space_group_info, n_sites=100, d_min=1.5):
    xrs = random_structure.xray_structure(
        space_group_info=space_group_info,
        elements=(("O", "N", "C") * (n_sites // 3 + 1))[:n_sites],
        volume_per_atom=50,
        min_distance=1.5,
    )
    f_obs = abs(xrs.structure_factors(d_min=d_min).f_calc())
    # exercise reciprocal_space_vector()
    for mi, d in zip(f_obs.indices(), f_obs.d_spacings().data()):
        rsv = flex.double(f_obs.unit_cell().reciprocal_space_vector(mi))
        assert approx_equal(d, 1.0 / math.sqrt(rsv.dot(rsv)))
    ##
    print f_obs.unit_cell()
    f = flex.random_double(f_obs.data().size()) * flex.mean(f_obs.data()) / 10
    #
    f_obs1 = f_obs.customized_copy(data=f_obs.data(), sigmas=f_obs.data() * f)
    print "datat in:", f_obs1.data().size()
    r = f_obs1.ellipsoidal_truncation_by_sigma(sigma_cutoff=1)
    print "data left:", r.data().size()
    r.miller_indices_as_pdb_file(file_name="indices1.pdb", expand_to_p1=False)
    r.miller_indices_as_pdb_file(file_name="indices2.pdb", expand_to_p1=True)
    #
    f_obs.miller_indices_as_pdb_file(file_name="indices3.pdb", expand_to_p1=False)
    f_obs.miller_indices_as_pdb_file(file_name="indices4.pdb", expand_to_p1=True)
    print "*" * 25
Пример #15
0
  def __init__(self, app, detector_id = None, detector = None, **kwargs):
    assert [detector_id, detector].count(None) == 1
    if detector is not None:
      kwargs['distance'] = flex.mean(flex.double([p.get_distance() for p in detector]))

    db_proxy.__init__(self, app, "%s_detector" % app.params.experiment_tag, id=detector_id, **kwargs)
    self.detector_id = self.id
Пример #16
0
def test1():

  dials_regression = libtbx.env.find_in_repositories(
    relative_path="dials_regression",
    test=os.path.isdir)

  data_dir = os.path.join(dials_regression, "centroid_test_data")
  datablock_path = os.path.join(data_dir, "datablock.json")

  # work in a temporary directory
  cwd = os.path.abspath(os.curdir)
  tmp_dir = open_tmp_directory(suffix="tst_rs_mapper")
  os.chdir(tmp_dir)
  cmd = 'dials.rs_mapper ' + datablock_path + ' map_file="junk.ccp4"'

  result = easy_run.fully_buffered(command=cmd).raise_if_errors()
  # load results
  from iotbx import ccp4_map
  from scitbx.array_family import flex
  m = ccp4_map.map_reader(file_name="junk.ccp4")
  assert len(m.data) == 7189057
  assert approx_equal(m.header_min, -1.0)
  assert approx_equal(flex.min(m.data), -1.0)

  assert approx_equal(m.header_max, 2052.75)
  assert approx_equal(flex.max(m.data), 2052.75)

  assert approx_equal(m.header_mean, 0.018606403842568398)
  assert approx_equal(flex.mean(m.data), 0.018606403842568398)

  print "OK"

  return
Пример #17
0
def blank_integrated_analysis(reflections, scan, phi_step, fractional_loss):
    prf_sel = reflections.get_flags(reflections.flags.integrated_prf)
    if prf_sel.count(True) > 0:
        reflections = reflections.select(prf_sel)
        intensities = reflections["intensity.prf.value"]
        variances = reflections["intensity.prf.variance"]
    else:
        sum_sel = reflections.get_flags(reflections.flags.integrated_sum)
        reflections = reflections.select(sum_sel)
        intensities = reflections["intensity.sum.value"]
        variances = reflections["intensity.sum.variance"]

    i_sigi = intensities / flex.sqrt(variances)

    xyz_px = reflections["xyzobs.px.value"]
    x_px, y_px, z_px = xyz_px.parts()
    phi = scan.get_angle_from_array_index(z_px)

    osc = scan.get_oscillation()[1]
    n_images_per_step = iceil(phi_step / osc)
    phi_step = n_images_per_step * osc

    phi_min = flex.min(phi)
    phi_max = flex.max(phi)
    n_steps = iceil((phi_max - phi_min) / phi_step)

    hist = flex.histogram(z_px, n_slots=n_steps)

    mean_i_sigi = flex.double()
    for i, slot_info in enumerate(hist.slot_infos()):
        sel = (z_px >= slot_info.low_cutoff) & (z_px < slot_info.high_cutoff)
        if sel.count(True) == 0:
            mean_i_sigi.append(0)
        else:
            mean_i_sigi.append(flex.mean(i_sigi.select(sel)))
    fractional_mean_i_sigi = mean_i_sigi / flex.max(mean_i_sigi)

    potential_blank_sel = mean_i_sigi <= (fractional_loss * flex.max(mean_i_sigi))

    xmin, xmax = zip(*[(slot_info.low_cutoff, slot_info.high_cutoff) for slot_info in hist.slot_infos()])

    d = {
        "data": [
            {
                "x": list(hist.slot_centers()),
                "y": list(mean_i_sigi),
                "xlow": xmin,
                "xhigh": xmax,
                "blank": list(potential_blank_sel),
                "type": "bar",
                "name": "blank_counts_analysis",
            }
        ],
        "layout": {"xaxis": {"title": "z observed (images)"}, "yaxis": {"title": "Number of reflections"}, "bargap": 0},
    }

    blank_regions = blank_regions_from_sel(d["data"][0])
    d["blank_regions"] = blank_regions

    return d
Пример #18
0
  def __init__(self,
               target_map,
               pdb_hierarchy,
               atom_radius,
               use_adp_restraints,
               nproc,
               log=None):
    adopt_init_args(self, locals())
    self.xray_structure = self.pdb_hierarchy.extract_xray_structure(
      crystal_symmetry = self.target_map.miller_array.crystal_symmetry())
    b_isos = self.xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)

    self.xray_structure = self.xray_structure.set_b_iso(
      value = flex.mean(b_isos))

    #for rg in self.pdb_hierarchy.residue_groups():
    #  sel = rg.atoms().extract_i_seq()
    #  sel = flex.bool(b_isos.size(), sel)
    #  self.xray_structure = self.xray_structure.set_b_iso(
    #    value     = flex.mean(b_isos.select(sel)),
    #    selection = sel)
    self.pdb_hierarchy.adopt_xray_structure(self.xray_structure)
    self.chain_selections = []
    for chain in self.pdb_hierarchy.chains():
      self.chain_selections.append(chain.atoms().extract_i_seq())
Пример #19
0
 def get_sites_cc (self, atoms, sites=None) :
   from cctbx import maptbx
   from scitbx.array_family import flex
   radii = flex.double()
   for atom in atoms :
     if (atom.element.strip() in ["H", "D"]) :
       radii.append(1.)
     else :
       radii.append(1.5)
   fcalc_map = self.fcalc_real_map
   if (sites is None) :
     sites = atoms.extract_xyz()
   else :
     fcalc_map = self.get_new_fcalc_map(
       sites_new=sites,
       i_seqs=atoms.extract_i_seq())
   sel = maptbx.grid_indices_around_sites(
     unit_cell  = self.unit_cell,
     fft_n_real = self.n_real,
     fft_m_real = self.m_real,
     sites_cart = sites,
     site_radii = radii)
   m1 = self.real_map.select(sel)
   m2 = fcalc_map.select(sel)
   cc = flex.linear_correlation(x=m1, y=m2).coefficient()
   return group_args(
     cc=cc,
     map_mean=flex.mean(m1.as_1d()))
Пример #20
0
 def residual_contribution(self, sites_current, sites_average):
   diff = sites_current - sites_average
   self.rms_with_respect_to_average.append(flex.mean(diff.dot())**0.5)
   self.number_of_restraints += diff.size()
   self.residual_sum += self.weight * diff.sum_sq()
   if (self.gradients is not None):
     return (2 * self.weight) * diff
def create_sigmas (f_obs, params, wilson_b=None, return_as_amplitudes=False) :
  assert (f_obs.sigmas() is None)
  from scitbx.array_family import flex
  i_obs = f_obs.f_as_f_sq()
  i_norm = i_obs.data() / flex.mean(i_obs.data())
  profiler = profile_sigma_generator(
    mtz_file=params.noise_profile_file,
    pdb_file=params.profile_model_file,
    wilson_b=wilson_b,
    data_label=params.profile_data_label,
    n_resolution_bins=params.n_resolution_bins,
    n_intensity_bins=params.n_intensity_bins)
  sigmas = flex.double(i_norm.size(), 0.0)
  i_obs.setup_binner(n_bins=params.n_resolution_bins)
  for j_bin in i_obs.binner().range_used() :
    bin_sel = i_obs.binner().selection(j_bin)
    shell_profile = profiler.get_noise_profile_for_shell(j_bin)
    for k in bin_sel.iselection() :
      i_over_sigma = shell_profile.get_i_over_sigma(i_norm[k])
      sigmas[k] = i_obs.data()[k] / i_over_sigma
  i_new = i_obs.customized_copy(sigmas=sigmas)
  if (return_as_amplitudes) :
    return i_new.f_as_f_sq()
  else :
    return i_new
Пример #22
0
 def update(self, xray_structure, accept_as_is=True):
   if(not accept_as_is):
     current_map = self.compute_map(xray_structure = xray_structure)
     sites_cart  = xray_structure.sites_cart()
     sites_cart_ = self.xray_structure.sites_cart()
     for r in self.residue_monitors:
       sca = sites_cart.select(r.selection_all)
       scs = sites_cart.select(r.selection_sidechain)
       scb = sites_cart.select(r.selection_backbone)
       map_cc_all       = self.map_cc(sites_cart = sca, other_map = current_map)
       map_cc_sidechain = self.map_cc(sites_cart = scs, other_map = current_map)
       map_cc_backbone  = self.map_cc(sites_cart = scb, other_map = current_map)
       flag = map_cc_all      >= r.map_cc_all and \
              map_cc_backbone >= r.map_cc_backbone and \
              map_cc_sidechain>= r.map_cc_sidechain
       if(flag):
         residue_sites_cart_new = sites_cart.select(r.selection_all)
         sites_cart_ = sites_cart_.set_selected(r.selection_all,
           residue_sites_cart_new)
     xray_structure = xray_structure.replace_sites_cart(sites_cart_)
   # re-initialize monitor
   self.dist_from_previous = flex.mean(self.xray_structure.distances(
     other = xray_structure))
   self.xray_structure = xray_structure
   self.pdb_hierarchy.adopt_xray_structure(xray_structure)
   self.initialize()
   self.states_collector.add(sites_cart = xray_structure.sites_cart())
   self.assert_pdb_hierarchy_xray_structure_sync()
Пример #23
0
 def box_iterator(self):
   b = maptbx.boxes(
     n_real   = self.atom_map_asu.focus(),
     fraction = self.box_size_as_fraction,
     max_boxes= self.max_boxes,
     log      = self.log)
   def get_wide_box(s,e): # define wide box: neutral + phased volumes
     if(self.neutral_volume_box_cushion_width>0):
       sh = self.neutral_volume_box_cushion_width
       ss = [max(s[i]-sh,0) for i in [0,1,2]]
       ee = [min(e[i]+sh,n_real_asu[i]) for i in [0,1,2]]
     else: ss,ee = s,e
     return ss,ee
   n_real_asu = b.n_real
   n_boxes = len(b.starts)
   i_box = 0
   for s,e in zip(b.starts, b.ends):
     i_box+=1
     sw,ew = get_wide_box(s=s,e=e)
     fmodel_omit = self.omit_box(start=sw, end=ew)
     r = fmodel_omit.r_work()
     self.r.append(r) # for tests only
     if(self.log):
       print >> self.log, "r(curr,min,max,mean)=%6.4f %6.4f %6.4f %6.4f"%(r,
         flex.min(self.r), flex.max(self.r), flex.mean(self.r)), i_box, n_boxes
     omit_map_data = self.asu_map_from_fmodel(
       fmodel=fmodel_omit, map_type=self.map_type)
     maptbx.copy_box(
       map_data_from = omit_map_data,
       map_data_to   = self.map_result_asu,
       start         = s,
       end           = e)
   self.map_result_asu.reshape(self.acc_asu)
def get_mean_statistic_for_resolution (d_min, stat_type, range=0.2, out=None) :
  if (out is None) :
    out = sys.stdout
  from scitbx.array_family import flex
  pkl_file = libtbx.env.find_in_repositories(
    relative_path = "chem_data/polygon_data/all_mvd.pickle",
    test = os.path.isfile)
  db = easy_pickle.load(pkl_file)
  all_d_min = db['high_resolution']
  stat_values = db[stat_type]
  values_for_range = flex.double()
  for (d_, v_) in zip(all_d_min, stat_values) :
    try :
      d = float(d_)
      v = float(v_)
    except ValueError : continue
    else :
      if (d > (d_min - range)) and (d < (d_min + range)) :
        values_for_range.append(v)
  h = flex.histogram(values_for_range, n_slots=10)
  print >> out, "  %s for d_min = %.3f - %.3f A" % (stat_names[stat_type], d_min-range,
    d_min+range)
  min = flex.min(values_for_range)
  max = flex.max(values_for_range)
  mean = flex.mean(values_for_range)
  print >> out, "    count: %d" % values_for_range.size()
  print >> out, "    min: %.2f" % min
  print >> out, "    max: %.2f" % max
  print >> out, "    mean: %.2f" % mean
  print >> out, "    histogram of values:"
  h.show(prefix="      ")
  return mean
def run(b_iso=10):
  xrs = iotbx.pdb.input(source_info=None, lines=pdb_str).xray_structure_simple()
  xrs = xrs.set_b_iso(value=b_iso)
  xrs.scattering_type_registry(
    table = "n_gaussian",
    d_min = 0.,
    types_without_a_scattering_contribution=["?"])
  n_real = [100,100,100]
  pixel_volume = xrs.unit_cell().volume()/(n_real[0]*n_real[1]*n_real[2])
  map_data_3d = mmtbx.real_space.sampled_model_density(
    xray_structure = xrs,
    n_real         = n_real).data()*pixel_volume
  dist, map_data_2d = maptbx.map_peak_3d_as_2d(
    map_data    = map_data_3d,
    unit_cell   = xrs.unit_cell(),
    center_cart = xrs.sites_cart()[0],
    radius      = 3.0)
  #
  map_data_2d_exact = flex.double()
  ed = xrs._scattering_type_registry.gaussian("Ca")
  for r in dist:
    map_data_2d_exact.append(ed.electron_density(r, b_iso))
  map_data_2d_exact = map_data_2d_exact * pixel_volume
  #
  assert approx_equal(flex.sum(map_data_3d), 20, 0.1) # Page 556, Int.Tables.
  assert flex.mean(abs(map_data_2d-map_data_2d_exact)) < 1.e-4
Пример #26
0
def finalize_model (pdb_hierarchy,
    xray_structure,
    set_b_iso=None,
    convert_to_isotropic=None,
    selection=None) :
  """
  Prepare a rebuilt model for refinement, optionally including B-factor reset.
  """
  from cctbx import adptbx
  from scitbx.array_family import flex
  pdb_atoms = pdb_hierarchy.atoms()
  if (selection is None) :
    selection = flex.bool(pdb_atoms.size(), True)
  elif isinstance(selection, str) :
    sel_cache = pdb_hierarchy.atom_selection_cache()
    selection = sel_cache.selection(selection)
  for i_seq, atom in enumerate(pdb_atoms) :
    assert (atom.parent() is not None)
    atom.segid = ""
    sc = xray_structure.scatterers()[i_seq]
    sc.label = atom.id_str()
  if (convert_to_isotropic) :
    xray_structure.convert_to_isotropic(selection=selection.iselection())
  if (set_b_iso is not None) :
    if (set_b_iso is Auto) :
      u_iso = xray_structure.extract_u_iso_or_u_equiv()
      set_b_iso = adptbx.u_as_b(flex.mean(u_iso)) / 2
    xray_structure.set_b_iso(value=set_b_iso, selection=selection)
  pdb_hierarchy.adopt_xray_structure(xray_structure)
  pdb_atoms.reset_serial()
  pdb_atoms.reset_i_seq()
  pdb_atoms.reset_tmp()
Пример #27
0
  def compute_functional_and_gradients(self):
    from scitbx.array_family import flex
    import math

    self.model_mean_x = flex.double(len(self.observed_x))
    self.model_mean_y = flex.double(len(self.observed_x))

    for x in xrange(6):
      selection = (self.master_groups==x)
      self.model_mean_x.set_selected(selection, self.x[2*x])
      self.model_mean_y.set_selected(selection, self.x[2*x+1])

    delx = self.observed_x - self.model_mean_x
    dely = self.observed_y - self.model_mean_y
    delrsq = delx*delx + dely*dely

    f = flex.sum(delrsq)

    gradients = flex.double([0.]*12)
    for x in xrange(6):
      selection = (self.master_groups==x)
      gradients[2*x] = -2. * flex.sum( delx.select(selection) )
      gradients[2*x+1]=-2. * flex.sum( dely.select(selection) )
    if self.verbose:
      print "Functional ",math.sqrt(flex.mean(delrsq))
    self.count_iterations += 1
    return f,gradients
Пример #28
0
    def compute_functional_and_gradients(self):
        # HATTNE does never enter this function
        print "HATTNE entering mark1.compute_functional_and_gradients"

        self.model_calcx = self.spotcx.deep_copy()
        self.model_calcy = self.spotcy.deep_copy()

        for x in xrange(64):
            selection = self.selections[x]
            self.model_calcx.set_selected(selection, self.model_calcx + self.x[2 * x])
            self.model_calcy.set_selected(selection, self.model_calcy + self.x[2 * x + 1])

        squares = self.delrsq_functional(self.model_calcx, self.model_calcy)

        f = flex.sum(squares)
        calc_obs_diffx = self.model_calcx - self.spotfx
        calc_obs_diffy = self.model_calcy - self.spotfy

        gradients = flex.double([0.0] * 128)
        for x in xrange(64):
            selection = self.selections[x]
            gradients[2 * x] = 2.0 * flex.sum(calc_obs_diffx.select(selection))
            gradients[2 * x + 1] = 2.0 * flex.sum(calc_obs_diffy.select(selection))
        print "Functional ", math.sqrt(flex.mean(squares))

        return f, gradients
Пример #29
0
def run():
  xrs = random_structure.xray_structure(
    space_group_info = sgtbx.space_group_info("P1"),
    elements         = ["N"]*500,
    unit_cell        = (20, 30, 40, 70, 80, 120))
  xrs = xrs.set_b_iso(value=25)
  d_mins = [1.5]
  result = []
  for d_min in d_mins:
    f_obs = abs(xrs.structure_factors(d_min=d_min).f_calc())
    f_obs = f_obs.customized_copy(data = f_obs.data() * 135.)
    shifts = [0.0, 0.3]
    for xyz_shake_amount in shifts:
      xrs_shaken = xrs.deep_copy_scatterers()
      xrs_shaken.shake_sites_in_place(mean_distance = xyz_shake_amount)
      ml_err = flex.double()
      ml_err_new = flex.double()
      for trial in xrange(10):
        r_free_flags = f_obs.generate_r_free_flags()
        fmodel = mmtbx.f_model.manager(
          f_obs          = f_obs,
          r_free_flags   = r_free_flags,
          xray_structure = xrs_shaken)
        ml_err_ = fmodel.model_error_ml()
        ml_err.append(ml_err_)
      result.append(flex.mean(ml_err))
  assert result[0] > 0 and result[0] < 0.03
  assert result[1] > 0.2 and result[1] <= 0.32
Пример #30
0
 def collect(O, rmsd_t_c, param_values):
   mins = [flex.min(a) for a in rmsd_t_c]
   means = [flex.mean(a) for a in rmsd_t_c]
   if (mins[0] < mins[1]): a = "t"
   else:                   a = "c"
   if (means[0] < means[1]): b = "t"
   else:                     b = "c"
   O.data[a+b].append(param_values)
Пример #31
0
    def __init__(self,
                 evaluator,
                 population_size=50,
                 f=None,
                 cr=0.9,
                 eps=1e-2,
                 n_cross=1,
                 max_iter=10000,
                 monitor_cycle=200,
                 out=None,
                 show_progress=False,
                 show_progress_nth_cycle=1,
                 insert_solution_vector=None,
                 dither_constant=0.4):
        self.dither = dither_constant
        self.show_progress = show_progress
        self.show_progress_nth_cycle = show_progress_nth_cycle
        self.evaluator = evaluator
        self.population_size = population_size
        self.f = f
        self.cr = cr
        self.n_cross = n_cross
        self.max_iter = max_iter
        self.monitor_cycle = monitor_cycle
        self.vector_length = evaluator.n
        self.eps = eps
        self.population = []
        self.seeded = False
        if insert_solution_vector is not None:
            assert len(insert_solution_vector) == self.vector_length
            self.seeded = insert_solution_vector
        for ii in xrange(self.population_size):
            self.population.append(flex.double(self.vector_length, 0))

        self.scores = flex.double(self.population_size, 1000)
        self.optimize()
        self.best_score = flex.min(self.scores)
        self.best_vector = self.population[flex.min_index(self.scores)]
        self.evaluator.x = self.best_vector
        if self.show_progress:
            self.evaluator.print_status(
                flex.min(self.scores), flex.mean(self.scores),
                self.population[flex.min_index(self.scores)], 'Final')
Пример #32
0
    def get_binned_intensities(self, n_bins=100):
        """
    Using self.ISIGI, bin the intensities using the following procedure:
    1) Find the minimum and maximum intensity values.
    2) Divide max-min by n_bins. This is the bin step size
    The effect is
    @param n_bins number of bins to use.
    @return a tuple with an array of selections for each bin and an array of median
    intensity values for each bin.
    """
        print >> self.log, "Computing intensity bins.",
        all_mean_Is = flex.double()
        only_means = flex.double()
        for hkl_id in range(self.scaler.n_refl):
            hkl = self.scaler.miller_set.indices()[hkl_id]
            if hkl not in self.scaler.ISIGI: continue
            n = len(self.scaler.ISIGI[hkl])
            # get scaled intensities
            intensities = flex.double(
                [self.scaler.ISIGI[hkl][i][0] for i in range(n)])
            meanI = flex.mean(intensities)
            only_means.append(meanI)
            all_mean_Is.extend(flex.double([meanI] * n))
        step = (flex.max(only_means) - flex.min(only_means)) / n_bins
        print >> self.log, "Bin size:", step

        sels = []
        binned_intensities = []
        min_all_mean_Is = flex.min(all_mean_Is)
        for i in range(n_bins):
            sel = (all_mean_Is >
                   (min_all_mean_Is + step * i)) & (all_mean_Is <
                                                    (min_all_mean_Is + step *
                                                     (i + 1)))
            if sel.all_eq(False): continue
            sels.append(sel)
            binned_intensities.append((step / 2 + step * i) + min(only_means))

        for i, (sel, intensity) in enumerate(zip(sels, binned_intensities)):
            print >> self.log, "Bin %02d, number of observations: % 10d, midpoint intensity: %f" % (
                i, sel.count(True), intensity)

        return sels, binned_intensities
  def __init__(self, tophat, normal, plot):
    # take U-mats from two different distributions, apply them to unit vectors, and plot
    if plot:
      from matplotlib import pyplot as plt
      fig, axes = plt.subplots(2, 2,figsize=(8,7))
    else:
      axes = ((1,2),(3,4)) #dummy

    # columns plot the transformation of x, y, and z unit vectors
    rows = [tophat,normal]
    differences = []
    for irow,dist in enumerate(rows):
      iaxes = axes[irow];
      cube_diag = math.sqrt(1./3) # 0.57735
      for icol, RLP in enumerate([(0,0,1), (cube_diag, cube_diag, cube_diag)]):
        RLP = col(RLP)
        print("(%7.5f %7.5f %7.5f)"%(RLP.elems),"Vector length:%8.6f"%(RLP.length()), end=' ')
        axis = iaxes[icol]
        unit = RLP.normalize()
        seed = col((1,0,0))
        perm2 = unit.cross(seed)
        perm3 = unit.cross(perm2)
        a2 = flex.double(); a3 = flex.double()
        difference_vectors = flex.vec3_double()
        for u in dist:
          U = sqr(u)
          newvec = U * RLP
          difference_vectors.append( newvec-RLP )
          a2.append(newvec.dot(perm2)); a3.append(newvec.dot(perm3))
        rms = math.sqrt( flex.mean ( difference_vectors.dot(difference_vectors) ) )
        print("The rms difference is", rms)
        differences.append(rms)
        if plot:
          axis.plot (a2,a3,'r,')
          axis.set_aspect("equal")
          axis.set_title("Transformation of v=%s"%("(%5.3f,%5.3f,%5.3f)"%(RLP.elems)))
          axis.set_xlim(-0.05,0.05)
          axis.set_ylim(-0.05,0.05)

      assert approx_equal(differences[0],differences[1],eps=1e-04), \
      "RMS mosaic distribution for axis vector and diagonal vector should be similar, as proposed by J Holton"

    if plot: plt.show()
Пример #34
0
    def get_active_data_percentile(self):
        data = self.imgobj.linearintdata
        indexing = []
        for asic in self.corners:
            block = data.matrix_copy_block(i_row=asic[0],
                                           i_column=asic[1],
                                           n_rows=asic[2] - asic[0],
                                           n_columns=asic[3] - asic[1])
            active_data = block.as_1d().as_double()

            order = flex.sort_permutation(active_data)
            if self.verbose:
                print("The mean is ", flex.mean(active_data),
                      "on %d pixels" % len(active_data))
                print("The 90-percentile pixel is ",
                      active_data[order[int(0.9 * len(active_data))]])
                print("The 99-percentile pixel is ",
                      active_data[order[int(0.99 * len(active_data))]])

            percentile90 = active_data[order[int(0.9 * len(active_data))]]
            maximas = flex.vec2_double()
            for idx in range(
                    len(active_data) - 1, int(0.9 * len(active_data)), -1):
                if active_data[order[idx]] > 2.0 * percentile90:
                    if self.verbose:
                        print("    ", idx, active_data[order[idx]])
                    irow = order[idx] // (asic[3] - asic[1])
                    icol = order[idx] % (asic[3] - asic[1])
                    #self.green.append((asic[0]+irow, asic[1]+icol))
                    maximas.append((irow, icol))
            CLUS = clustering(maximas)
            #coords = CLUS.as_spot_max_pixels(block,asic)
            coords = CLUS.as_spot_center_of_mass(block, asic, percentile90)
            intensities = CLUS.intensities
            for coord, height in zip(coords, intensities):
                self.green.append(coord)
                indexing.append((
                    coord[0] * float(self.inputpd["pixel_size"]),
                    coord[1] * float(self.inputpd["pixel_size"]),
                    0.0,  # 0 -degree offset for still image
                    height))
        return indexing
def calc_monte_carlo_population_covariances(mats, mean_matrix):
    """Given the sequence of matrices mats, calculate the variance-covariance
  matrix of the elements, using the known mean values for each elt in mat
  to avoid the approximation implied in taking the sample covariance"""

    # check input
    assert all([e.is_square() for e in mats])
    n = mats[0].n_rows()
    assert all([e.n_rows() == n for e in mats])

    # create an empty var-cov matrix
    covmat = flex.double(flex.grid(n**2, n**2), 0.0)

    for i in range(covmat.all()[0]):
        for j in range(covmat.all()[1]):
            resid_a = flex.double([m[i] - mean_matrix[i] for m in mats])
            resid_b = flex.double([m[j] - mean_matrix[j] for m in mats])
            covmat[i, j] = flex.mean(resid_a * resid_b)

    return covmat
Пример #36
0
 def initialize(self):
     self.assert_pdb_hierarchy_xray_structure_sync()
     five_cc_o = five_cc(map=self.target_map_object.map_data,
                         xray_structure=self.xray_structure,
                         d_min=self.target_map_object.d_min,
                         compute_cc_box=True,
                         compute_cc_image=False,
                         compute_cc_mask=True,
                         compute_cc_volume=False,
                         compute_cc_peaks=False)
     self.cc_mask = five_cc_o.cc_mask
     self.cc_box = five_cc_o.cc_box
     if (self.geometry_restraints_manager is not None):
         es = self.geometry_restraints_manager.energies_sites(
             sites_cart=self.xray_structure.sites_cart())
         self.rmsd_a = es.angle_deviations()[2]
         self.rmsd_b = es.bond_deviations()[2]
     self.dist_from_start = flex.mean(
         self.xray_structure_start.distances(other=self.xray_structure))
     self.assert_pdb_hierarchy_xray_structure_sync()
Пример #37
0
def run(space_group_info):
  """
  Make sure it work for all space groups and boxes with non-zero origin.
  """
  # make up data
  xrs = random_structure.xray_structure(
    space_group_info       = space_group_info,
    volume_per_atom        = 50,
    general_positions_only = False,
    u_iso                  = 0.3,
    elements               = ('C', 'N', 'O', "S")*10,
    min_distance           = 1.5)
  xrs.scattering_type_registry(table="wk1995")
  f_calc = xrs.structure_factors(d_min=2).f_calc()
  f_obs = abs(f_calc)
  # create fmodel object
  fmodel = mmtbx.f_model.manager(
    xray_structure = xrs,
    f_obs          = f_obs)
  fmodel.update_all_scales()
  mc1 = fmodel.electron_density_map().map_coefficients(
      map_type     = "2mFo-DFc",
      isotropize   = False,
      exclude_free_r_reflections=False,
      fill_missing = False)
  crystal_gridding = fmodel.f_obs().crystal_gridding(
    d_min             = fmodel.f_obs().d_min(),
    symmetry_flags    = maptbx.use_space_group_symmetry,
    resolution_factor = 1./3)
  # compute OMIT map
  r = cfom.run(
    crystal_gridding    = crystal_gridding,
    fmodel              = fmodel.deep_copy(),
    full_resolution_map = False,
    max_boxes           = 70,
    neutral_volume_box_cushion_width = 0,
    box_size_as_fraction=0.3,
    log=False)
  ccs = get_cc(mc1=mc1, mc2=r.map_coefficients(filter_noise=False), xrs=xrs)
  assert flex.mean(ccs) > 0.8
  print("  CC(min/max,mean)",ccs.min_max_mean().as_tuple())
Пример #38
0
def exercise():
  # Exercise "simple" target
  pi = get_pdb_inputs(pdb_str=pdb_str_1)
  selection = flex.bool(pi.xrs.scatterers().size(), True)
  for d_min in [1, 2, 3]:
    print "d_min:", d_min
    f_calc = pi.xrs.structure_factors(d_min = d_min).f_calc()
    fft_map = f_calc.fft_map(resolution_factor=0.25)
    fft_map.apply_sigma_scaling()
    target_map = fft_map.real_map_unpadded()
    rsr_simple_refiner = individual_sites.simple(
      target_map                  = target_map,
      selection                   = selection,
      real_space_gradients_delta  = d_min/4,
      max_iterations              = 150,
      geometry_restraints_manager = pi.grm.geometry)
    for shake_size in [1,]:
      print "  shake_size:", shake_size
      for p in [(0.01, 1.0), (0.03, 3.0), (0.1, 10.0)]:
        print "    target:", p
        w_opt = flex.double()
        for start_value in [0.001, 0.01, 0.1, 0, 1, 10, 100, 1000]:
          xrs_poor = pi.xrs.deep_copy_scatterers()
          random.seed(0)
          flex.set_random_seed(0)
          xrs_poor.shake_sites_in_place(mean_distance = shake_size)
          #
          refined = individual_sites.refinery(
            refiner                  = rsr_simple_refiner,
            xray_structure           = xrs_poor,
            start_trial_weight_value = start_value,
            rms_bonds_limit          = p[0],
            rms_angles_limit         = p[1])
          w_opt.append(refined.weight_final)
          dist = flex.mean(flex.sqrt((pi.xrs.sites_cart() -
            refined.sites_cart_result).dot()))
          print "      w_start,w_final,b,a,dist: %9.4f %9.4f %6.3f %6.3f %6.3f"%(
            start_value, refined.weight_final, refined.rms_bonds_final,
            refined.rms_angles_final, dist)
          assert refined.rms_bonds_final  <= p[0]
          assert refined.rms_angles_final <= p[1]
Пример #39
0
def followup_brightness_scale(data):
    """ histogramming ported from iotbx/detectors/display.h """
    #first pass through data calculate average
    data = data.as_double()
    qave = flex.mean(data)

    #second pass calculate histogram
    hsize = 100

    histogram = flex.double(hsize, 0)
    for i in xrange(data.size()):
        temp = int((hsize / 2) * data[i] / qave)
        if temp < 0: histogram[0] += 1
        elif temp >= hsize: histogram[hsize - 1] += 1
        else: histogram[temp] += 1

    #third pass calculate 90%
    percentile = 0
    accum = 0
    for i in xrange(hsize):
        accum += histogram[i]
        if (accum > 0.9 * data.size()):
            percentile = i * qave / (hsize / 2)
            break

    adjlevel = 0.4
    brightness = 0.4
    if percentile > 0.:
        correction = brightness * adjlevel / percentile
    else:
        correction = brightness / 5.0

    outscale = 256
    corrected = data * correction
    outvalue = outscale * (1.0 - corrected)
    sel1 = outvalue < 0
    sel2 = outvalue >= outscale
    outvalue.set_selected(sel1, 0)
    outvalue.set_selected(sel2, outscale - 1)

    return outvalue
Пример #40
0
def show_histogram(data, n_slots, smooth=True):
    triplets = []
    histogram = flex.histogram(data=data, n_slots=n_slots)
    l = histogram.data_min()
    for i, s in enumerate(histogram.slots()):
        r = histogram.data_min() + histogram.slot_width() * (i + 1)
        triplets.append([l, r, s])
        print "%8.4f %8.4f %d" % (l, r, s)
        l = r
    if (smooth):
        print "... smooth histogram"
        triplets_smooth = []
        for i, t in enumerate(triplets):
            values = flex.double()
            for j in [-1, 0, 1]:
                if (i + j >= 0 and i + j < len(triplets)):
                    values.append(float(triplets[i + j][2]))
            triplets_smooth.append((t[0], t[1], flex.mean(values)))
        for t in triplets_smooth:
            print "%8.4f %8.4f %d" % (t[0], t[1], int("%.0f" % t[2]))
    return histogram
Пример #41
0
 def do_sigma_scaling(self):
   # Divide each pixel value by it's dark standard deviation.  Since we are led
   # to believe that the standard deviation of a pixel is proportional to the
   # gain of said pixel, this approximates a gain correction.
   assert self.dark_img is not None
   assert self.gain_map is None # not appropriate to do sigma scaling and gain correction at the same time!
   flex_cspad_img = self.cspad_img.as_double()
   flex_cspad_img_sel = flex_cspad_img.as_1d().select(self.dark_mask.as_1d())
   flex_dark_stddev = self.dark_stddev.select(self.dark_mask.as_1d()).as_double()
   assert flex_dark_stddev.count(0) == 0
   flex_dark_stddev /= flex.mean(flex_dark_stddev)
   flex_cspad_img_sel /= flex_dark_stddev
   flex_cspad_img.as_1d().set_selected(self.dark_mask.as_1d().iselection(), flex_cspad_img_sel)
   self.cspad_img = flex_cspad_img
   if 0: # for debugging
     from matplotlib import pyplot
     hist_min, hist_max = flex.min(flex_cspad_img_sel.as_double()), flex.max(flex_cspad_img_sel.as_double())
     print hist_min, hist_max
     n_slots = 100
     n, bins, patches = pyplot.hist(flex_cspad_img_sel.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
     pyplot.show()
Пример #42
0
def estimate_signal_to_noise(x, y_noisy, y_smoothed, plot=False):
    """Estimate noise in spectra by subtracting a smoothed spectrum from the
     original noisy unsmoothed spectrum.

     See:
       The extraction of signal to noise values in x-ray absorption spectroscopy
       A. J. Dent, P. C. Stephenson, and G. N. Greaves
       Rev. Sci. Instrum. 63, 856 (1992); https://doi.org/10.1063/1.1142627
  """
    noise = y_noisy - y_smoothed
    noise_sq = flex.pow2(noise)
    from xfel.command_line.view_pixel_histograms import sliding_average
    sigma_sq = sliding_average(noise_sq, n=31)
    sigma_sq = smoothing.savitzky_golay_filter(x.as_double(),
                                               flex.pow2(noise),
                                               half_window=20,
                                               degree=1)[1]
    sigma_sq.set_selected(sigma_sq <= 0, flex.mean(sigma_sq))
    # or do this instead to use the background region as the source of noise:
    #signal_to_noise = y_smoothed/math.sqrt(flex.mean(noise_sq[50:190]))
    signal_to_noise = y_smoothed / flex.sqrt(sigma_sq)
    #signal_to_noise.set_selected(x < 50, 0)
    #signal_to_noise.set_selected(x > 375, 0)
    if plot:
        from matplotlib import pyplot
        linewidth = 2
        pyplot.plot(x, y_noisy, linewidth=linewidth)
        pyplot.plot(x, y_smoothed, linewidth=linewidth)
        pyplot_label_axes()
        pyplot.show()
        pyplot.plot(x, noise, linewidth=linewidth, label="noise")
        pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=linewidth, label="sigma")
        pyplot_label_axes()
        pyplot.legend(loc=2, prop={'size': 20})
        pyplot.show()
        pyplot.plot(x, signal_to_noise, linewidth=linewidth)
        pyplot_label_axes()
        pyplot.show()

    return signal_to_noise
Пример #43
0
def reduce_raw_data(raw_data, qmax, bandwidth, level=0.05, q_background=None):
    print
    print " ====  Data reduction ==== "
    print
    print "  Preprocessing of data increases efficiency of shape retrieval procedure."
    print
    print "   -  Interpolation stepsize                           :  %4.3e" % bandwidth
    print "   -  Uniform density criteria:  level is set to       :  %4.3e" % level
    print "                                 maximum q to consider :  %4.3e" % qmax
    qmin_indx = flex.max_index(raw_data.i)
    qmin = raw_data.q[qmin_indx]
    new_data = get_q_array_uniform_body(raw_data,
                                        q_min=qmin,
                                        q_max=qmax,
                                        level=level)
    qmax = new_data.q[-1]
    if qmax > raw_data.q[-1]:
        qmax = raw_data.q[-1]
    print "      Resulting q range to use in  search:   q start   :  %4.3e" % qmin
    print "                                             q stop    :  %4.3e" % qmax
    print
    raw_q = raw_data.q[qmin_indx:]
    raw_i = raw_data.i[qmin_indx:]
    raw_s = raw_data.s[qmin_indx:]
    ### Take care of the background (set zero at very high q) ###
    if (q_background is not None):
        cutoff = flex.bool(raw_q > q_background)
        q_bk_indx = flex.last_index(cutoff, False)
        if (q_bk_indx < raw_q.size()):
            bkgrd = flex.mean(raw_i[q_bk_indx:])
            print "Background correction: I=I-background, where background=", bkgrd
            raw_i = flex.abs(raw_i - bkgrd)

    q = flex.double(range(int(
        (qmax - qmin) / bandwidth) + 1)) * bandwidth + qmin
    raw_data.i = flex.linear_interpolation(raw_q, raw_i, q)
    raw_data.s = flex.linear_interpolation(raw_q, raw_s, q)
    raw_data.q = q

    return raw_data
Пример #44
0
def construct_src_to_dst_plan(icount, tranch_size, comm, verbose=True):
      mpi_rank = comm.Get_rank()
      mpi_size = comm.Get_size()
      COMPOSITE_MULTIPLICITY = 3
      from scitbx.array_family import flex
      imean = flex.mean(icount.as_double()); isum = flex.sum(icount)
      trial_comm_size = max(1, int(isum * COMPOSITE_MULTIPLICITY / tranch_size))
      trial_comm_size = min(trial_comm_size, mpi_size - 1) # new comm must have fewer than mpi_helper.size ranks
      if trial_comm_size in [2,3]: trial_comm_size = 1 # consensus communicator can run on 1,4,5,..., not 2,3
      trial_comm_size = max(1, trial_comm_size) # cannot be 0

      if verbose: print("recommended communicator size:",trial_comm_size)
      trial = [[] for itrank in range(trial_comm_size)]
      srcrk = [[] for itrank in range(trial_comm_size)]
      todst = {} # plan of action: src rank values to dst keys
      mt = flex.mersenne_twister(seed=0)
      order = mt.random_permutation(len(icount))
      available_dst = list(range(trial_comm_size))
      for idx in range(len(icount)):
        item = icount[order[idx]] ; rk = idx
        if trial_comm_size>3:
          unavailable = []
          for irx in range(COMPOSITE_MULTIPLICITY):
            if len(available_dst)==0:  available_dst = list(range(trial_comm_size))
            idx_st = choose_without_replace(available_dst, unavailable, mt)
            unavailable.append(idx_st)
            trial[(idx_st)%trial_comm_size].append(item); srcrk[(idx_st)%trial_comm_size].append(rk)
            todst[idx_st] = todst.get(idx_st,[]); todst[idx_st].append(order[idx])
        else:
          trial[0].append(item)
          idx_st = 0
          todst[idx_st] = todst.get(idx_st,[]); todst[idx_st].append(order[idx])
      if verbose:
        for itranch, item in enumerate(trial):
          print("tranch %2d:"%itranch, item, flex.sum(flex.int(item)))
        print()
        for item in srcrk:
          print(", ".join(["%02d"%i for i in item]))
        print(todst)
      return todst
Пример #45
0
def get_atom_radius(xray_structure=None, d_min=None, map_data=None,
                    crystal_symmetry=None, radius=None):
  if(radius is not None): return radius
  radii = []
  if(d_min is not None):
    radii.append(d_min)
  if([xray_structure, crystal_symmetry].count(None)==0):
    assert crystal_symmetry.is_similar_symmetry(
      xray_structure.crystal_symmetry())
  if([map_data, crystal_symmetry].count(None)==0):
    d99 = maptbx.d99(
      map              = map_data,
      crystal_symmetry = crystal_symmetry).result.d99
    radii.append(d99)
  if(xray_structure is not None and d_min is not None):
    b_iso = adptbx.u_as_b(
      flex.mean(xray_structure.extract_u_iso_or_u_equiv()))
    o = maptbx.atom_curves(scattering_type="C", scattering_table="electron")
    rad_image = o.image(d_min=d_min, b_iso=b_iso,
      radius_max=max(15.,d_min), radius_step=0.01).radius
    radii.append(rad_image)
  return max(3, min(10, max(radii)))
Пример #46
0
    def pair_align(self):
        ms = flex.double()
        ss = flex.double()
        tmp_nlm_array = math.nlm_array(self.nmax)
        for coef in self.finals:
            mean = abs(coef[0])
            var = flex.sum(flex.norm(coef))
            sigma = smath.sqrt(var - mean * mean)
            ms.append(mean)
            ss.append(sigma)

        grids = flex.grid(self.n_trial, self.n_trial)
        self.cc_array = flex.double(grids, 1.0)
        for ii in range(self.n_trial):
            self.nlm_array.load_coefs(self.nlm, self.finals[ii])
            for jj in range(ii):
                tmp_nlm_array.load_coefs(self.nlm, self.finals[jj])
                cc = fft_align.align(self.nlm_array,
                                     tmp_nlm_array,
                                     nmax=self.nmax,
                                     refine=True).best_score
                cc = (cc - ms[ii] * ms[jj]) / (ss[ii] * ss[jj])
                self.cc_array[(ii, jj)] = cc
                self.cc_array[(jj, ii)] = cc

        outfile = self.prefix + "pair.cc"
        comment = "# electron density correlation coefficient, < rho_1(r)*rho_2(r) >"
        out = open(outfile, 'w')
        print >> out, comment
        for ii in range(1, self.n_trial + 1):
            print >> out, "%6d" % ii,
        print >> out, "   average"

        for ii in range(self.n_trial):
            for jj in range(self.n_trial):
                print >> out, "%6.3f" % self.cc_array[(ii, jj)],
            print >> out, flex.mean(self.cc_array[ii * self.n_trial:(ii + 1) *
                                                  self.n_trial])
        out.close()
Пример #47
0
    def compute(self):
        """
    Computes the reduced chi squared parameter. Can be used to
    correct for under-estimation of experimental errors.
    See
    https://en.wikipedia.org/wiki/Weighted_arithmetic_mean#Correcting_for_over-_or_under-dispersion
    """
        print >> self.log, "Computing reduced chi squared"
        self.scaler.reduced_chi_squared = flex.double(self.scaler.n_refl, 1.)

        for hkl_id in range(self.scaler.n_refl):
            hkl = self.scaler.miller_set.indices()[hkl_id]
            if hkl not in self.scaler.ISIGI: continue

            n = len(self.scaler.ISIGI[hkl])
            if n <= 1:
                continue

            i = self.scaler.summed_wt_I[hkl_id] / self.scaler.summed_weight[
                hkl_id]
            x = flex.double([self.scaler.ISIGI[hkl][i][0] for i in range(n)])
            v = (x /
                 flex.double([self.scaler.ISIGI[hkl][i][1]
                              for i in range(n)]))**2

            self.scaler.reduced_chi_squared[hkl_id] = 1 / (n - 1) * flex.sum(
                (x - i)**2 / v)
        sel = self.scaler.reduced_chi_squared > 0

        print >> self.log, "Done computing reduced chi squared", flex.mean(
            self.scaler.reduced_chi_squared.select(sel))

        if False:
            from matplotlib import pyplot as plt
            plt.hist(
                self.scaler.reduced_chi_squared.select(sel).as_numpy_array(),
                bins=100,
                range=(0, 10))
            plt.show()
Пример #48
0
 def show(self, prefix="", out=sys.stdout):
     if (self.n_residues() == 1):
         print(prefix + "Segment: 1 residue (%s), %d conformers" % \
           (self.residue_groups[0].id_str(), self.n_confs()), file=out)
     else:
         print(prefix+"Segment: %d residues (%s --> %s), %d conformers" %\
           (self.n_residues(), self.residue_groups[0].id_str(),
            self.residue_groups[-1].id_str(), self.n_confs()), file=out)
     for i_res, rg in enumerate(self.residue_groups):
         print(prefix + "  residue_group=%s" % rg.id_str(), file=out)
         for ag in rg.atom_groups():
             rama = rota = None
             for o in self.ramachandran.get(rg.id_str(), []):
                 if (o.altloc == ag.altloc):
                     rama = o
                     break
             for o in self.rotamers.get(rg.id_str(), []):
                 if (o.altloc == ag.altloc):
                     rota = o
                     break
             print(prefix + "    " + \
               "atom_group=%1s %3s  occ=%.2f phi=%-6s psi=%-6s rot=%-7s" %\
               (ag.altloc,
                ag.resname,
                flex.mean(ag.atoms().extract_occ()),
                fv("%.1f", getattr(rama, "phi", None)),
                fv("%.1f", getattr(rama, "psi", None)),
                getattr(rota, "rotamer_name", None)), file=out)
         if (len(self.backrubs[i_res]) > 0):
             for backrub in self.backrubs[i_res]:
                 backrub.show(out=out, prefix=prefix + "    ")
         outliers = self.outliers[rg.id_str()]
         if (len(outliers) > 0):
             print(prefix + "     MolProbity outliers:", file=out)
         for outlier in outliers:
             print(prefix + "       %s: %s" %
                   (type(outlier).__name__, str(outlier)),
                   file=out)
Пример #49
0
    def box_iterator(self):
        b = maptbx.boxes(n_real=self.atom_map_asu.focus(),
                         fraction=self.box_size_as_fraction,
                         max_boxes=self.max_boxes,
                         log=self.log)

        def get_wide_box(s, e):  # define wide box: neutral + phased volumes
            if (self.neutral_volume_box_cushion_width > 0):
                sh = self.neutral_volume_box_cushion_width
                ss = [max(s[i] - sh, 0) for i in [0, 1, 2]]
                ee = [min(e[i] + sh, n_real_asu[i]) for i in [0, 1, 2]]
            else:
                ss, ee = s, e
            return ss, ee

        n_real_asu = b.n_real
        n_boxes = len(b.starts)
        i_box = 0
        for s, e in zip(b.starts, b.ends):
            i_box += 1
            sw, ew = get_wide_box(s=s, e=e)
            fmodel_omit = self.omit_box(start=sw, end=ew)
            r = fmodel_omit.r_work()
            self.r.append(r)  # for tests only
            if (self.log):
                print(
                    "r(curr,min,max,mean)=%6.4f %6.4f %6.4f %6.4f" %
                    (r, flex.min(self.r), flex.max(self.r), flex.mean(self.r)),
                    i_box,
                    n_boxes,
                    file=self.log)
            omit_map_data = self.asu_map_from_fmodel(fmodel=fmodel_omit,
                                                     map_type=self.map_type)
            maptbx.copy_box(map_data_from=omit_map_data,
                            map_data_to=self.map_result_asu,
                            start=s,
                            end=e)
        self.map_result_asu.reshape(self.acc_asu)
Пример #50
0
def get_mean_statistic_for_resolution(d_min,
                                      stat_type,
                                      range_value=0.2,
                                      out=None):
    if (out is None):
        out = sys.stdout
    from scitbx.array_family import flex
    pkl_file = libtbx.env.find_in_repositories(
        relative_path="chem_data/polygon_data/all_mvd.pickle",
        test=os.path.isfile)
    db = easy_pickle.load(pkl_file)
    all_d_min = db['high_resolution']
    stat_values = db[stat_type]
    values_for_range = flex.double()
    for (d_, v_) in zip(all_d_min, stat_values):
        try:
            d = float(d_)
            v = float(v_)
        except ValueError:
            continue
        else:
            if (d > (d_min - range_value)) and (d < (d_min + range_value)):
                values_for_range.append(v)
    h = flex.histogram(values_for_range, n_slots=10)
    print("  %s for d_min = %.3f - %.3f A" %
          (stat_names[stat_type], d_min - range_value, d_min + range_value),
          file=out)
    min = flex.min(values_for_range)
    max = flex.max(values_for_range)
    mean = flex.mean(values_for_range)
    print("    count: %d" % values_for_range.size(), file=out)
    print("    min: %.2f" % min, file=out)
    print("    max: %.2f" % max, file=out)
    print("    mean: %.2f" % mean, file=out)
    print("    histogram of values:", file=out)
    h.show(prefix="      ")
    return mean
Пример #51
0
def find_duplicate_conformers(residue_group, rmsd_cutoff=0.1):
    """Return pairs of residue objects for conformers with an RMSD less than rmsd_cutoff"""

    rmsd_cutoff_sq = rmsd_cutoff**2
    duplicate_conformers = []
    for i, c1 in enumerate(residue_group.conformers()):
        r1 = c1.only_residue()
        a1 = r1.atoms()
        a1_nam = list(a1.extract_name())
        for j, c2 in enumerate(residue_group.conformers()):
            if j <= i:
                continue
            # Extract residue and skip if not comparable
            r2 = c2.only_residue()
            if r1.resname != r2.resname:
                continue
            # Extract atoms
            a2 = r2.atoms()
            a2_nam = list(a2.extract_name())
            # Get atom overlap between conformers
            common_atoms = list(set(a1_nam).intersection(a2_nam))
            # Sort the atoms so can use unsorted residues
            a1_sel = flex.size_t([a1_nam.index(an) for an in common_atoms])
            a2_sel = flex.size_t([a2_nam.index(an) for an in common_atoms])
            # Check selection working as it should
            assert a1.extract_name().select(
                a1_sel) == a2.extract_name().select(a2_sel)
            # Extract ordered coordinates
            a1_xyz = a1.extract_xyz().select(a1_sel)
            a2_xyz = a2.extract_xyz().select(a2_sel)
            # Claculate RMSD and check below threshold
            d = flex.mean((a1_xyz - a2_xyz).dot())
            if d < rmsd_cutoff_sq:
                duplicate_conformers.append(
                    (r1.standalone_copy(), r2.standalone_copy()))

    return duplicate_conformers
Пример #52
0
def test_rs_mapper(dials_data, tmp_path):
    result = procrunner.run(
        [
            "dials.rs_mapper",
            dials_data("centroid_test_data", pathlib=True) /
            "imported_experiments.json",
            'map_file="junk.ccp4"',
        ],
        working_directory=tmp_path,
    )
    assert not result.returncode and not result.stderr
    assert (tmp_path / "junk.ccp4").is_file()

    # load results
    m = ccp4_map.map_reader(file_name=str(tmp_path / "junk.ccp4"))
    assert len(m.data) == 7189057
    assert m.header_min == 0.0
    assert flex.min(m.data) == 0.0

    assert m.header_max == 2052.75
    assert flex.max(m.data) == 2052.75

    assert m.header_mean == pytest.approx(0.018924040719866753, abs=1e-6)
    assert flex.mean(m.data) == pytest.approx(0.01892407052218914, abs=1e-6)
Пример #53
0
def run(prefix):
    """
  Exercise "qr.finalise m.pdb" produces expected (and meaningful) output.
  Do not modify this test before checking the result on graphics (eg, PyMol)!
  """
    pdb_in = "%s.pdb" % prefix
    open(pdb_in, "w").write(pdb_str_in)
    cmd = "qr.finalise %s > %s.log" % (pdb_in, prefix)
    assert easy_run.call(cmd) == 0
    h_answer = iotbx.pdb.input(source_info=None,
                               lines=pdb_str_out).construct_hierarchy()
    h_result = iotbx.pdb.input(file_name="%s_complete.pdb" %
                               prefix).construct_hierarchy()
    #
    s1 = h_answer.atoms().extract_xyz()
    s2 = h_result.atoms().extract_xyz()
    r = flex.mean(flex.sqrt((s1 - s2).dot()))
    assert r < 0.005
    #
    asc = h_result.atom_selection_cache()
    sel = asc.selection("element H or element D")
    assert sel.count(True) == 7
    occ = h_result.atoms().extract_occ().select(sel)
    assert flex.max(occ) < 1.e-6
Пример #54
0
 def get_sites_cc(self, atoms, sites=None):
     from cctbx import maptbx
     from scitbx.array_family import flex
     radii = flex.double()
     for atom in atoms:
         if (atom.element.strip() in ["H", "D"]):
             radii.append(1.)
         else:
             radii.append(1.5)
     fcalc_map = self.fcalc_real_map
     if (sites is None):
         sites = atoms.extract_xyz()
     else:
         fcalc_map = self.get_new_fcalc_map(sites_new=sites,
                                            i_seqs=atoms.extract_i_seq())
     sel = maptbx.grid_indices_around_sites(unit_cell=self.unit_cell,
                                            fft_n_real=self.n_real,
                                            fft_m_real=self.m_real,
                                            sites_cart=sites,
                                            site_radii=radii)
     m1 = self.real_map.select(sel)
     m2 = fcalc_map.select(sel)
     cc = flex.linear_correlation(x=m1, y=m2).coefficient()
     return group_args(cc=cc, map_mean=flex.mean(m1.as_1d()))
Пример #55
0
def log_frame(experiments,
              reflections,
              params,
              run,
              n_strong,
              timestamp=None,
              two_theta_low=None,
              two_theta_high=None,
              db_event=None,
              app=None,
              trial=None):
    if app is None:
        app = dxtbx_xfel_db_application(params, mode='cache_commits')
    else:
        app.mode = 'cache_commits'

    if isinstance(run, int) or isinstance(run, str):
        db_run = app.get_run(run_number=run)
    else:
        db_run = run

    if trial is None:
        if params.input.trial is None:
            db_trial = app.get_trial(trial_id=params.input.trial_id)
            params.input.trial = db_trial.trial
        else:
            db_trial = app.get_trial(trial_number=params.input.trial)
    else:
        db_trial = trial

    if db_event is None:
        if params.input.rungroup is None:
            db_event = app.create_event(timestamp=timestamp,
                                        run_id=db_run.id,
                                        trial_id=db_trial.id,
                                        n_strong=n_strong,
                                        two_theta_low=two_theta_low,
                                        two_theta_high=two_theta_high)
        else:
            db_event = app.create_event(timestamp=timestamp,
                                        run_id=db_run.id,
                                        trial_id=db_trial.id,
                                        rungroup_id=params.input.rungroup,
                                        n_strong=n_strong,
                                        two_theta_low=two_theta_low,
                                        two_theta_high=two_theta_high)

    inserts = ""

    if app.last_query is None:
        app.last_query = ""

    def save_last_id(name):
        nonlocal inserts
        inserts += app.last_query + ";\n"
        inserts += "SELECT LAST_INSERT_ID() INTO @%s_id;\n" % name

    if experiments:
        save_last_id('event')
    else:
        inserts += app.last_query + ";\n"

    for i, experiment in enumerate(experiments or []):
        reflections_i = reflections.select(reflections['id'] == i)

        imageset = Imageset(app)
        save_last_id('imageset')

        beam = Beam(app, beam=experiment.beam)
        save_last_id('beam')

        detector = Detector(app, detector=experiment.detector)
        save_last_id('detector')

        cell = Cell(app, crystal=experiment.crystal, isoform_id=None)
        save_last_id('cell')

        crystal = Crystal(app,
                          crystal=experiment.crystal,
                          make_cell=False,
                          cell_id="@cell_id")
        save_last_id('crystal')

        inserts += ("INSERT INTO `%s_experiment` (imageset_id, beam_id, detector_id, crystal_id, crystal_cell_id) " + \
                    "VALUES (@imageset_id, @beam_id, @detector_id, @crystal_id, @cell_id);\n") % (
          params.experiment_tag)

        inserts += "INSERT INTO `%s_imageset_event` (imageset_id, event_id, event_run_id) VALUES (@imageset_id, @event_id, %d);\n" % (
            params.experiment_tag, db_run.id)

        d = experiment.crystal.get_unit_cell().d(
            reflections['miller_index']).select(reflections['id'] == i)
        from cctbx.crystal import symmetry
        cs = symmetry(unit_cell=experiment.crystal.get_unit_cell(),
                      space_group=experiment.crystal.get_space_group())
        mset = cs.build_miller_set(anomalous_flag=False, d_min=db_trial.d_min)
        n_bins = 10  # FIXME use n_bins as an attribute on the trial table
        binner = mset.setup_binner(n_bins=n_bins)
        for i in binner.range_used():
            d_max, d_min = binner.bin_d_range(i)
            Bin(app,
                number=i,
                d_min=d_min,
                d_max=d_max,
                total_hkl=binner.counts_complete()[i],
                cell_id='@cell_id')
            save_last_id('bin')

            sel = (d <= float(d_max)) & (d > float(d_min))
            sel &= reflections_i['intensity.sum.value'] > 0
            refls = reflections_i.select(sel)
            n_refls = len(refls)
            Cell_Bin(
                app,
                count=n_refls,
                bin_id='@bin_id',
                crystal_id='@crystal_id',
                avg_intensity=flex.mean(refls['intensity.sum.value'])
                if n_refls > 0 else None,
                avg_sigma=flex.mean(flex.sqrt(refls['intensity.sum.variance']))
                if n_refls > 0 else None,
                avg_i_sigi=flex.mean(
                    refls['intensity.sum.value'] /
                    flex.sqrt(refls['intensity.sum.variance']))
                if n_refls > 0 else None)
            inserts += app.last_query + ";\n"
    app.mode = 'execute'
    return inserts
Пример #56
0
def run (args, image = None):
  from xfel import radial_average
  from scitbx.array_family import flex
  import os, sys
  import dxtbx

  # Parse input
  try:
    n = len(args)
  except Exception:
    params = args
  else:
    user_phil = []
    for arg in args:
      if (not "=" in arg):
        try :
          user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg))
        except ValueError:
          raise Sorry("Unrecognized argument '%s'" % arg)
      else:
        try:
          user_phil.append(libtbx.phil.parse(arg))
        except RuntimeError as e:
          raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
    params = master_phil.fetch(sources=user_phil).extract()
  if image is None:
    if params.file_path is None or len(params.file_path) == 0 or not all([os.path.isfile(f) for f in params.file_path]):
      master_phil.show()
      raise Usage("file_path must be defined (either file_path=XXX, or the path alone).")
  assert params.n_bins is not None
  assert params.verbose is not None
  assert params.output_bins is not None

  # Allow writing to a file instead of stdout
  if params.output_file is None:
    logger = sys.stdout
  else:
    logger = open(params.output_file, 'w')
    logger.write("%s "%params.output_file)

  if params.show_plots:
    from matplotlib import pyplot as plt
    import numpy as np
    colormap = plt.cm.gist_ncar
    plt.gca().set_color_cycle([colormap(i) for i in np.linspace(0, 0.9, len(params.file_path))])

  if params.mask is not None:
    params.mask = easy_pickle.load(params.mask)

  if image is None:
    iterable = params.file_path
    load_func = lambda x: dxtbx.load(x)
  else:
    iterable = [image]
    load_func = lambda x: x

  # Iterate over each file provided
  for item in iterable:
    img = load_func(item)
    try:
      n_images = img.get_num_images()
      subiterable = xrange(n_images)
    except AttributeError:
      n_images = None
      subiterable = [0]
    for image_number in subiterable:
      if n_images is None:
        beam = img.get_beam()
        detector = img.get_detector()
      else:
        beam = img.get_beam(image_number)
        detector = img.get_detector(image_number)
      s0 = col(beam.get_s0())

      # Search the detector for the panel farthest from the beam. The number of bins in the radial average will be
      # equal to the farthest point from the beam on the detector, in pixels, unless overridden at the command line
      panel_res = [p.get_max_resolution_at_corners(s0) for p in detector]
      farthest_panel = detector[panel_res.index(min(panel_res))]
      size2, size1 = farthest_panel.get_image_size()
      corners = [(0,0), (size1-1,0), (0,size2-1), (size1-1,size2-1)]
      corners_lab = [col(farthest_panel.get_pixel_lab_coord(c)) for c in corners]
      corner_two_thetas = [farthest_panel.get_two_theta_at_pixel(s0, c) for c in corners]
      extent_two_theta = max(corner_two_thetas)
      max_corner = corners_lab[corner_two_thetas.index(extent_two_theta)]
      extent = int(math.ceil(max_corner.length()*math.sin(extent_two_theta)/max(farthest_panel.get_pixel_size())))
      extent_two_theta *= 180/math.pi

      if params.n_bins < extent:
        params.n_bins = extent

      # These arrays will store the radial average info
      sums    = flex.double(params.n_bins) * 0
      sums_sq = flex.double(params.n_bins) * 0
      counts  = flex.int(params.n_bins) * 0

      if n_images is None:
        all_data = img.get_raw_data()
      else:
        all_data = img.get_raw_data(image_number)

      if not isinstance(all_data, tuple):
        all_data = (all_data,)

      for tile, (panel, data) in enumerate(zip(detector, all_data)):
        if params.mask is None:
          mask = flex.bool(flex.grid(data.focus()), True)
        else:
          mask = params.mask[tile]

        if hasattr(data,"as_double"):
          data = data.as_double()

        logger.flush()
        if params.verbose:
          logger.write("Average intensity tile %d: %9.3f\n"%(tile, flex.mean(data)))
          logger.write("N bins: %d\n"%params.n_bins)
          logger.flush()

        x1,y1,x2,y2 = 0,0,panel.get_image_size()[1],panel.get_image_size()[0]
        bc = panel.get_beam_centre_px(beam.get_s0())
        bc = int(round(bc[1])), int(round(bc[0]))

        # compute the average
        radial_average(data,mask,bc,sums,sums_sq,counts,panel.get_pixel_size()[0],panel.get_distance(),
                       (x1,y1),(x2,y2))

      # average the results, avoiding division by zero
      results = sums.set_selected(counts <= 0, 0)
      results /= counts.set_selected(counts <= 0, 1).as_double()

      if params.median_filter_size is not None:
        logger.write("WARNING, the median filter is not fully propogated to the variances\n")
        from scipy.ndimage.filters import median_filter
        results = flex.double(median_filter(results.as_numpy_array(), size = params.median_filter_size))

      # calculate standard devations
      stddev_sel = ((sums_sq-sums*results) >= 0) & (counts > 0)
      std_devs = flex.double(len(sums), 0)
      std_devs.set_selected(stddev_sel,
                           (sums_sq.select(stddev_sel)-sums.select(stddev_sel)* \
                            results.select(stddev_sel))/counts.select(stddev_sel).as_double())
      std_devs = flex.sqrt(std_devs)

      twotheta = flex.double(xrange(len(results)))*extent_two_theta/params.n_bins
      q_vals = 4*math.pi*flex.sin(math.pi*twotheta/360)/beam.get_wavelength()

      if params.low_max_two_theta_limit is None:
        subset = results
      else:
        subset = results.select(twotheta >= params.low_max_two_theta_limit)

      max_result = flex.max(subset)

      if params.x_axis == 'two_theta':
        xvals = twotheta
        max_x = twotheta[flex.first_index(results, max_result)]
      elif params.x_axis == 'q':
        xvals = q_vals
        max_x = q_vals[flex.first_index(results, max_result)]

      for i in xrange(len(results)):
        val = xvals[i]
        if params.output_bins and "%.3f"%results[i] != "nan":
         #logger.write("%9.3f %9.3f\n"%     (val,results[i]))        #.xy  format for Rex.cell.
          logger.write("%9.3f %9.3f %9.3f\n"%(val,results[i],std_devs[i])) #.xye format for GSASII
         #logger.write("%.3f %.3f %.3f\n"%(val,results[i],ds[i]))  # include calculated d spacings
      logger.write("Maximum %s: %f, value: %f\n"%(params.x_axis, max_x, max_result))

      if params.show_plots:
        if params.plot_x_max is not None:
          results = results.select(xvals <= params.plot_x_max)
          xvals = xvals.select(xvals <= params.plot_x_max)
        if params.normalize:
          plt.plot(xvals.as_numpy_array(),(results/flex.max(results)).as_numpy_array(),'-')
        else:
          plt.plot(xvals.as_numpy_array(),results.as_numpy_array(),'-')
        if params.x_axis == 'two_theta':
          plt.xlabel("2 theta")
        elif params.x_axis == 'q':
          plt.xlabel("q")
        plt.ylabel("Avg ADUs")
        if params.plot_y_max is not None:
          plt.ylim(0, params.plot_y_max)

    if params.show_plots:
      #plt.legend([os.path.basename(os.path.splitext(f)[0]) for f in params.file_path], ncol=2)
      plt.show()

  return xvals, results
Пример #57
0
 def _avg_sd_from_list(lst):
     """simple function to get average and standard deviation"""
     arr = flex.double(lst)
     avg = round(flex.mean(arr), 5)
     std = round(arr.standard_deviation_of_the_sample(), 5)
     return avg, std
Пример #58
0
    def adjust_errors(self):
        print >> self.log, "Starting adjust_errors"

        # Save original sigmas
        refls = self.scaler.ISIGI
        refls['original_sigmas'] = refls['scaled_intensity'] / refls['isigi']

        print >> self.log, "Computing initial estimates of parameters"
        propagator = sdfac_propagate(self.scaler, verbose=False)
        propagator.initial_estimates()
        propagator.adjust_errors(compute_sums=False)
        init_params = flex.double(self.get_initial_sdparams_estimates())
        init_params.extend(propagator.error_terms.to_x())
        values = self.parameterization(init_params)

        print >> self.log, "Initial estimates:",
        values.show(self.log)
        print >> self.log, "Refining error correction parameters"
        sels, binned_intensities = self.get_binned_intensities()
        minimizer = self.run_minimzer(values, sels)
        values = minimizer.get_refined_params()
        print >> self.log, "Final",
        values.show(self.log)

        print >> self.log, "Applying sdfac/sdb/sdadd 1"
        # Restore original sigmas
        refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas']

        # Propagate refined errors from postrefinement
        propagator.error_terms = error_terms.from_x(values.propagate_terms)
        propagator.adjust_errors()
        minimizer.apply_sd_error_params(self.scaler.ISIGI, values)

        self.scaler.summed_weight = flex.double(self.scaler.n_refl, 0.)
        self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.)

        print >> self.log, "Applying sdfac/sdb/sdadd 2"
        for i in xrange(len(self.scaler.ISIGI)):
            hkl_id = self.scaler.ISIGI['miller_id'][i]
            Intensity = self.scaler.ISIGI['scaled_intensity'][
                i]  # scaled intensity
            sigma = Intensity / self.scaler.ISIGI['isigi'][
                i]  # corrected sigma
            variance = sigma * sigma
            self.scaler.summed_wt_I[hkl_id] += Intensity / variance
            self.scaler.summed_weight[hkl_id] += 1 / variance

        if False:
            # validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations
            print >> self.log, "Validating"
            from matplotlib import pyplot as plt
            all_sigmas_normalized = compute_normalized_deviations(
                self.scaler.ISIGI, self.scaler.miller_set.indices())
            plt.hist(all_sigmas_normalized, bins=100)
            plt.figure()

            binned_rms_normalized_sigmas = []

            for i, sel in enumerate(sels):
                binned_rms_normalized_sigmas.append(
                    math.sqrt(
                        flex.mean(
                            all_sigmas_normalized.select(sel) *
                            all_sigmas_normalized.select(sel))))

            plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o')
            plt.show()

            all_sigmas_normalized = all_sigmas_normalized.select(
                all_sigmas_normalized != 0)
            self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5),
                                         plot=True)
Пример #59
0
 def initialize_A(self):
   # initial fit is to set the average
   self.A = self.gain * flex.mean(self.KI)
Пример #60
0
def log_frame(experiments,
              reflections,
              params,
              run,
              n_strong,
              timestamp=None,
              two_theta_low=None,
              two_theta_high=None,
              db_event=None):
    app = dxtbx_xfel_db_application(params)
    db_run = app.get_run(run_number=run)
    if params.input.trial is None:
        db_trial = app.get_trial(trial_id=params.input.trial_id)
        params.input.trial = db_trial.trial
    else:
        db_trial = app.get_trial(trial_number=params.input.trial)

    if db_event is None:
        if params.input.rungroup is None:
            db_event = app.create_event(timestamp=timestamp,
                                        run_id=db_run.id,
                                        trial_id=db_trial.id,
                                        n_strong=n_strong,
                                        two_theta_low=two_theta_low,
                                        two_theta_high=two_theta_high)
        else:
            db_event = app.create_event(timestamp=timestamp,
                                        run_id=db_run.id,
                                        trial_id=db_trial.id,
                                        rungroup_id=params.input.rungroup,
                                        n_strong=n_strong,
                                        two_theta_low=two_theta_low,
                                        two_theta_high=two_theta_high)

    for i, experiment in enumerate(experiments or []):
        reflections_i = reflections.select(reflections['id'] == i)
        db_experiment = app.create_experiment(experiment)
        app.link_imageset_frame(db_experiment.imageset, db_event)

        d = experiment.crystal.get_unit_cell().d(
            reflections['miller_index']).select(reflections['id'] == i)
        if len(
                db_experiment.crystal.cell.bins
        ) == 0:  # will be [] if there are no isoforms and no target cells
            from cctbx.crystal import symmetry
            cs = symmetry(
                unit_cell=db_experiment.crystal.cell.unit_cell,
                space_group_symbol=db_experiment.crystal.cell.lookup_symbol)
            mset = cs.build_miller_set(anomalous_flag=False,
                                       d_min=db_trial.d_min)
            n_bins = 10  # FIXME use n_bins as an attribute on the trial table
            binner = mset.setup_binner(n_bins=n_bins)
            for i in binner.range_used():
                d_max, d_min = binner.bin_d_range(i)
                Bin(app,
                    number=i,
                    d_min=d_min,
                    d_max=d_max,
                    total_hkl=binner.counts_complete()[i],
                    cell_id=db_experiment.crystal.cell.id)
            db_experiment.crystal.cell.bins = app.get_cell_bins(
                db_experiment.crystal.cell.id)
            assert len(db_experiment.crystal.cell.bins) == n_bins

        for db_bin in db_experiment.crystal.cell.bins:
            sel = (d <= float(db_bin.d_max)) & (d > float(db_bin.d_min))
            sel &= reflections_i['intensity.sum.value'] > 0
            refls = reflections_i.select(sel)
            n_refls = len(refls)
            Cell_Bin(
                app,
                count=n_refls,
                bin_id=db_bin.id,
                crystal_id=db_experiment.crystal.id,
                avg_intensity=flex.mean(refls['intensity.sum.value'])
                if n_refls > 0 else None,
                avg_sigma=flex.mean(flex.sqrt(refls['intensity.sum.variance']))
                if n_refls > 0 else None,
                avg_i_sigi=flex.mean(
                    refls['intensity.sum.value'] /
                    flex.sqrt(refls['intensity.sum.variance']))
                if n_refls > 0 else None)
    return db_event