コード例 #1
0
  def exercise_ls_cycles(self):
    xs = self.xray_structure.deep_copy_scatterers()
    connectivity_table = smtbx.utils.connectivity_table(xs)
    emma_ref = xs.as_emma_model()
    # shaking must happen before the reparametrisation is constructed,
    # otherwise the original values will prevail
    xs.shake_sites_in_place(rms_difference=0.1)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    cycles = normal_eqns_solving.naive_iterations(
      ls,
      n_max_iterations=5,
      track_all=True)

    assert approx_equal(ls.scale_factor(), 1, eps=1e-5)
    assert approx_equal(ls.objective(), 0)
    # skip next-to-last one to allow for no progress and rounding error
    assert (
      cycles.objective_history[0]
      >= cycles.objective_history[1]
      >= cycles.objective_history[3]), numstr(cycles.objective_history)
    assert approx_equal(cycles.gradient_norm_history[-1], 0, eps=5e-8)

    match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0]
    assert match.rt.r == matrix.identity(3)
    for pair in match.pairs:
      assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4)
コード例 #2
0
  def exercise_ls_cycles(self):
    xs = self.xray_structure.deep_copy_scatterers()
    connectivity_table = smtbx.utils.connectivity_table(xs)
    emma_ref = xs.as_emma_model()
    # shaking must happen before the reparametrisation is constructed,
    # otherwise the original values will prevail
    xs.shake_sites_in_place(rms_difference=0.1)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.mainstream_shelx_weighting(a=0),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    cycles = normal_eqns_solving.naive_iterations(
      ls,
      gradient_threshold=1e-12,
      step_threshold=1e-7,
      track_all=True)

    assert approx_equal(ls.scale_factor(), 1, eps=1e-5), ls.scale_factor()
    assert approx_equal(ls.objective(), 0), ls.objective()

    match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0]
    assert match.rt.r == matrix.identity(3)
    for pair in match.pairs:
      assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4)
コード例 #3
0
def exercise_symmetry_equivalent():
    xs = xray.structure(crystal_symmetry=crystal.symmetry(
        unit_cell=(1, 2, 3), space_group_symbol='hall: P 2x'),
                        scatterers=flex.xray_scatterer(
                            (xray.scatterer("C", site=(0.1, 0.2, 0.3)), )))
    xs.scatterers()[0].flags.set_grad_site(True)
    connectivity_table = smtbx.utils.connectivity_table(xs)
    reparametrisation = constraints.reparametrisation(xs, [],
                                                      connectivity_table)
    site_0 = reparametrisation.add(constraints.independent_site_parameter,
                                   scatterer=xs.scatterers()[0])
    g = sgtbx.rt_mx('x,-y,-z')
    symm_eq = reparametrisation.add(
        constraints.symmetry_equivalent_site_parameter, site=site_0, motion=g)
    reparametrisation.finalise()

    assert approx_equal(symm_eq.original.scatterers[0].site, (0.1, 0.2, 0.3),
                        eps=1e-15)
    assert str(symm_eq.motion) == 'x,-y,-z'
    assert symm_eq.is_variable
    reparametrisation.linearise()
    assert approx_equal(symm_eq.value, g * site_0.value, eps=1e-15)

    reparametrisation.store()
    assert approx_equal(symm_eq.value, (0.1, -0.2, -0.3), eps=1e-15)
    assert approx_equal(site_0.value, (0.1, 0.2, 0.3), eps=1e-15)
コード例 #4
0
def test_structures(sizes, d_mins):
    for na in sizes:
        xs = random_xray_structure(
            space_group_symbol='hall: -P 2ybc',
            n_scatterers=na,
            proportion_of_elements={
                'C': 5,
                'O': 2,
                'N': 1
            },
            use_u_iso=False,
            use_u_aniso=True,
        )
        for d_min in d_mins:
            mi = xs.build_miller_set(anomalous_flag=False, d_min=d_min)
            ma = mi.structure_factors_from_scatterers(
                xs, algorithm='direct').f_calc()
            fo_sq = ma.norm().customized_copy(
                sigmas=flex.double(ma.size(), 1.))
            xs.shake_sites_in_place(rms_difference=0.1)
            xs.shake_adp()
            for sc in xs.scatterers():
                sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
                sc.flags.set_grad_site(True).set_grad_u_aniso(True)
            connectivity_table = smtbx.utils.connectivity_table(xs)
            reparametrisation = constraints.reparametrisation(
                structure=xs,
                constraints=[],
                connectivity_table=connectivity_table)
            yield (fo_sq.as_xray_observations(), reparametrisation)
コード例 #5
0
def exercise_symmetry_equivalent():
  xs = xray.structure(
    crystal_symmetry=crystal.symmetry(
      unit_cell=(1, 2, 3),
      space_group_symbol='hall: P 2x'),
    scatterers=flex.xray_scatterer((
      xray.scatterer("C", site=(0.1, 0.2, 0.3)),
    )))
  xs.scatterers()[0].flags.set_grad_site(True)
  connectivity_table = smtbx.utils.connectivity_table(xs)
  reparametrisation = constraints.reparametrisation(
    xs, [], connectivity_table)
  site_0 = reparametrisation.add(constraints.independent_site_parameter,
                                 scatterer=xs.scatterers()[0])
  g = sgtbx.rt_mx('x,-y,-z')
  symm_eq = reparametrisation.add(
    constraints.symmetry_equivalent_site_parameter,
    site=site_0, motion=g)
  reparametrisation.finalise()

  assert approx_equal(symm_eq.original.scatterers[0].site, (0.1, 0.2, 0.3),
                      eps=1e-15)
  assert str(symm_eq.motion) == 'x,-y,-z'
  assert symm_eq.is_variable
  reparametrisation.linearise()
  assert approx_equal(symm_eq.value, g*site_0.value, eps=1e-15)

  reparametrisation.store()
  assert approx_equal(symm_eq.value, (0.1, -0.2, -0.3), eps=1e-15)
  assert approx_equal(site_0.value, (0.1, 0.2, 0.3), eps=1e-15)
コード例 #6
0
  def exercise_ls_cycles(self):
    xs = self.xray_structure.deep_copy_scatterers()
    connectivity_table = smtbx.utils.connectivity_table(xs)
    emma_ref = xs.as_emma_model()
    # shaking must happen before the reparametrisation is constructed,
    # otherwise the original values will prevail
    xs.shake_sites_in_place(rms_difference=0.1)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.mainstream_shelx_weighting(a=0),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    cycles = normal_eqns_solving.naive_iterations(
      ls,
      gradient_threshold=1e-12,
      step_threshold=1e-7,
      track_all=True)

    assert approx_equal(ls.scale_factor(), 1, eps=1e-5), ls.scale_factor()
    assert approx_equal(ls.objective(), 0), ls.objective()

    match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0]
    assert match.rt.r == matrix.identity(3)
    for pair in match.pairs:
      assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4)
コード例 #7
0
    def run(self):
        print("[ %s ]" % self.__class__.__name__)
        self.connectivity_table = smtbx.utils.connectivity_table(
            self.xray_structure)
        if not self.fpfdp_run:
            for sc in self.xray_structure.scatterers():
                sc.flags.set_grad_site(True)
                if sc.flags.use_u_aniso(): sc.flags.set_grad_u_aniso(True)
                if sc.flags.use_u_iso(): sc.flags.set_grad_u_iso(True)
        else:
            self.xray_structure.set_inelastic_form_factors(1.54184, 'sasaki')
            for sc in self.xray_structure.scatterers():
                sc.flags.set_grad_site(False)
                sc.flags.set_grad_u_iso(False)
                sc.flags.set_grad_u_aniso(False)
                sc.flags.set_grad_occupancy(False)
                if sc.element_symbol() == 'O':
                    sc.flags.set_use_fp_fdp(True)
                    sc.flags.set_grad_fp(True)
                    sc.flags.set_grad_fdp(True)

        self.reparametrisation = constraints.reparametrisation(
            self.xray_structure,
            self.constraints,
            self.connectivity_table,
            temperature=self.t_celsius,
        )

        self.check_reparametrisation_construction()
        self.check_mapping_to_grad_fc()

        # above settings are customised in the following tests
        # n.b. the fp,fdp tests override the default check_refinement_stability
        self.check_refinement_stability()
コード例 #8
0
  def exercise_ls_cycles(self):
    xs = self.xray_structure.deep_copy_scatterers()
    xs.shake_adp() # it must happen before the reparamtrisation is constructed
                   # because the ADP values are read then and only then.
    connectivity_table = smtbx.utils.connectivity_table(xs)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=oop.null())

    cycles = normal_eqns_solving.naive_iterations(
      ls,
      n_max_iterations=10,
      track_all=True)

    assert approx_equal(ls.scale_factor(), 1, eps=1e-4)
    assert approx_equal(ls.objective(), 0)
    # skip next-to-last one to allow for no progress and rounding error
    n = len(cycles.objective_history)
    assert cycles.objective_history[0] > cycles.objective_history[n-1],\
           cycles.objective_history
    assert approx_equal(cycles.gradient_norm_history[-1], 0, eps=1e-6)

    for sc0, sc1 in zip(self.xray_structure.scatterers(), xs.scatterers()):
      assert approx_equal(sc0.u_star, sc1.u_star)
コード例 #9
0
 def least_squares(self):
     reparametrisation = constraints.reparametrisation(
         self.xray_structure, self.constraints, self.connectivity_table)
     return least_squares.crystallographic_ls(
         self.fo_sq,
         reparametrisation,
         restraints_manager=self.restraints_manager,
         weighting_scheme=self.weighting_scheme)
コード例 #10
0
    def exercise(self, fixed_twin_fraction):
        # Create a shaken structure xs ready for refinement
        xs0 = self.structure
        emma_ref = xs0.as_emma_model()
        xs = xs0.deep_copy_scatterers()
        xs.shake_sites_in_place(rms_difference=0.15)
        xs.shake_adp()
        for sc in xs.scatterers():
            sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
            sc.flags.set_grad_site(True).set_grad_u_aniso(True)

        # Setup L.S. problem
        connectivity_table = smtbx.utils.connectivity_table(xs)
        shaken_twin_fraction = (self.twin_fraction if fixed_twin_fraction else
                                self.twin_fraction +
                                0.1 * flex.random_double())
        # 2nd domain in __init__
        twin_components = (xray.twin_component(twin_law=self.twin_law.r(),
                                               value=shaken_twin_fraction,
                                               grad=not fixed_twin_fraction), )
        reparametrisation = constraints.reparametrisation(
            structure=xs,
            constraints=[],
            connectivity_table=connectivity_table,
            twin_fractions=twin_components)
        obs = self.fo_sq.as_xray_observations(twin_components=twin_components)
        ls = least_squares.crystallographic_ls(
            obs,
            reparametrisation,
            weighting_scheme=least_squares.unit_weighting(),
            origin_fixing_restraints_type=origin_fixing_restraints.
            atomic_number_weighting)

        # Refine till we get back the original structure (so we hope)
        cycles = normal_eqns_solving.levenberg_marquardt_iterations(
            ls, gradient_threshold=1e-12, step_threshold=1e-6, track_all=True)

        # Now let's start to check it all worked
        assert ls.n_parameters == 63 if fixed_twin_fraction else 64

        match = emma.model_matches(emma_ref,
                                   xs.as_emma_model()).refined_matches[0]
        assert match.rt.r == matrix.identity(3)
        for pair in match.pairs:
            assert approx_equal(match.calculate_shortest_dist(pair),
                                0,
                                eps=1e-4), pair

        if fixed_twin_fraction:
            assert ls.twin_fractions[0].value == self.twin_fraction
        else:
            assert approx_equal(ls.twin_fractions[0].value,
                                self.twin_fraction,
                                eps=1e-2)

        assert approx_equal(ls.scale_factor(), 1, eps=1e-5)
        assert approx_equal(ls.objective(), 0)
コード例 #11
0
ファイル: __init__.py プロジェクト: cctbx/cctbx-playground
 def least_squares(self):
   reparametrisation = constraints.reparametrisation(
     self.xray_structure,
     self.constraints,
     self.connectivity_table)
   return least_squares.crystallographic_ls(
     self.fo_sq,
     reparametrisation,
     restraints_manager=self.restraints_manager,
     weighting_scheme=self.weighting_scheme)
コード例 #12
0
  def exercise(self, fixed_twin_fraction):
    # Create a shaken structure xs ready for refinement
    xs0 = self.structure
    emma_ref = xs0.as_emma_model()
    xs = xs0.deep_copy_scatterers()
    xs.shake_sites_in_place(rms_difference=0.15)
    xs.shake_adp()
    for sc in xs.scatterers():
      sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
      sc.flags.set_grad_site(True).set_grad_u_aniso(True)

    # Setup L.S. problem
    connectivity_table = smtbx.utils.connectivity_table(xs)
    shaken_twin_fraction = (
      self.twin_fraction if fixed_twin_fraction else
      self.twin_fraction + 0.1*flex.random_double())
    # 2nd domain in __init__
    twin_components = (xray.twin_component(
        twin_law=self.twin_law.r(), value=shaken_twin_fraction,
        grad=not fixed_twin_fraction),)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table,
      twin_fractions=twin_components)
    obs = self.fo_sq.as_xray_observations(twin_components=twin_components)
    ls = least_squares.crystallographic_ls(
      obs, reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    # Refine till we get back the original structure (so we hope)
    cycles = normal_eqns_solving.levenberg_marquardt_iterations(
      ls,
      gradient_threshold=1e-12,
      step_threshold=1e-6,
      track_all=True)

    # Now let's start to check it all worked
    assert ls.n_parameters == 63 if fixed_twin_fraction else 64

    match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0]
    assert match.rt.r == matrix.identity(3)
    for pair in match.pairs:
      assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4), pair

    if fixed_twin_fraction:
      assert ls.twin_fractions[0].value == self.twin_fraction
    else:
      assert approx_equal(ls.twin_fractions[0].value, self.twin_fraction,
                          eps=1e-2)

    assert approx_equal(ls.scale_factor(), 1, eps=1e-5)
    assert approx_equal(ls.objective(), 0)
コード例 #13
0
 def ls_problem():
   xs = xs1.deep_copy_scatterers()
   reparametrisation = constraints.reparametrisation(
     structure=xs,
     constraints=[],
     connectivity_table=smtbx.utils.connectivity_table(xs),
     temperature=20)
   return least_squares.crystallographic_ls(
     fo_sq.as_xray_observations(),
     reparametrisation=reparametrisation,
     restraints_manager=restraints_manager)
コード例 #14
0
 def ls_problem():
     xs = xs1.deep_copy_scatterers()
     reparametrisation = constraints.reparametrisation(
         structure=xs,
         constraints=[],
         connectivity_table=smtbx.utils.connectivity_table(xs),
         temperature=20)
     return least_squares.crystallographic_ls(
         fo_sq.as_xray_observations(),
         reparametrisation=reparametrisation,
         restraints_manager=restraints_manager)
コード例 #15
0
    def check_refinement_stability(self):
        if not self.shall_refine_thermal_displacements:
            for sc in self.xray_structure.scatterers():
                sc.flags.set_grad_site(True)
                if sc.flags.use_u_aniso(): sc.flags.set_grad_u_aniso(False)
                if sc.flags.use_u_iso(): sc.flags.set_grad_u_iso(False)

        xs = self.xray_structure
        xs0 = self.reference_xray_structure = xs.deep_copy_scatterers()
        mi = xs0.build_miller_set(anomalous_flag=False, d_min=0.5)
        fo_sq = mi.structure_factors_from_scatterers(
            xs0, algorithm="direct").f_calc().norm()
        fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))

        xs.shake_sites_in_place(rms_difference=0.1)
        if self.shall_refine_thermal_displacements:
            # a spread of 10 for u_iso's would be enormous for our low temperature
            # test structures if those u_iso's were not constrained
            xs.shake_adp(
                spread=10,  # absolute
                aniso_spread=0.2)  # relative

        self.reparametrisation = constraints.reparametrisation(
            xs,
            self.constraints,
            self.connectivity_table,
            temperature=self.t_celsius)
        obs = fo_sq.as_xray_observations()
        ls = least_squares.crystallographic_ls(
            obs,
            self.reparametrisation,
            weighting_scheme=least_squares.mainstream_shelx_weighting())
        self.cycles = self.normal_eqns_solving_method(ls)
        print("%i %s iterations to recover from shaking" %
              (self.cycles.n_iterations, self.cycles))
        if 0:
            from crys3d.qttbx.xray_structure_viewer import display
            display(xray_structure=xs)

        diff = xray.meaningful_site_cart_differences(xs0, xs)
        assert diff.max_absolute() < self.site_refinement_tolerance,\
               self.__class__.__name__

        if self.shall_refine_thermal_displacements:
            delta_u = []
            for sc, sc0 in itertools.izip(xs.scatterers(), xs0.scatterers()):
                if not sc.flags.use_u_aniso() or not sc0.flags.use_u_aniso():
                    continue
                delta_u.extend(matrix.col(sc.u_star) - matrix.col(sc0.u_star))
            delta_u = flex.double(delta_u)

            assert flex.max_absolute(delta_u) < self.u_star_refinement_tolerance,\
                   self.__class__.__name__
コード例 #16
0
    def check_refinement_stability(self):
        xs = self.xray_structure
        xs0 = self.reference_xray_structure = xs.deep_copy_scatterers()

        # First we construct the Fo array
        mi = xs0.build_miller_set(anomalous_flag=True, d_min=0.5)
        fo_sq = mi.structure_factors_from_scatterers(
            xs0, algorithm="direct").f_calc().norm()
        fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))

        # Then shake adps and store the shaken adp components
        self.shake_selection = flex.bool(
            [sc.element_symbol() in ['C', 'O'] for sc in xs.scatterers()])
        xs.shake_adp(selection=self.shake_selection)
        adp_ref = []
        for sc in xs.scatterers():
            if sc.flags.use_u_aniso():
                adp_ref.extend(sc.u_star)
            else:
                adp_ref.append(sc.u_iso)

        # Then do the refinement
        self.reparametrisation = constraints.reparametrisation(
            xs,
            self.constraints,
            self.connectivity_table,
            temperature=self.t_celsius)
        self.obs = fo_sq.as_xray_observations()
        ls = least_squares.crystallographic_ls(
            self.obs,
            self.reparametrisation,
            weighting_scheme=least_squares.mainstream_shelx_weighting())
        self.cycles = self.normal_eqns_solving_method(ls)
        print("%i %s iterations to recover from shaking" %
              (self.cycles.n_iterations, self.cycles))

        # Then verify the final ADPs have all changed by the same amount
        adp_final = []
        for sc in xs.scatterers():
            if sc.flags.use_u_aniso():
                adp_final.extend(sc.u_star)
            else:
                adp_final.append(sc.u_iso)
        adp_ratios = [
            x / y for x, y in zip(adp_ref, adp_final) if x > 1e-6 and y > 1e-6
        ]
        assert all(
            [approx_equal(x, adp_ratios[0], eps=1e-9) for x in adp_ratios[1:]])

        #Make sure the ADPs have changed by some non-zero amount
        assert adp_ratios[0] > 1.01 or adp_ratios[0] < 0.99
コード例 #17
0
  def check_refinement_stability(self):
    if not self.shall_refine_thermal_displacements:
      for sc in self.xray_structure.scatterers():
        sc.flags.set_grad_site(True)
        if sc.flags.use_u_aniso(): sc.flags.set_grad_u_aniso(False)
        if sc.flags.use_u_iso(): sc.flags.set_grad_u_iso(False)

    xs = self.xray_structure
    xs0 = self.reference_xray_structure = xs.deep_copy_scatterers()
    mi = xs0.build_miller_set(anomalous_flag=False, d_min=0.5)
    fo_sq = mi.structure_factors_from_scatterers(
      xs0, algorithm="direct").f_calc().norm()
    fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))

    xs.shake_sites_in_place(rms_difference=0.1)
    if self.shall_refine_thermal_displacements:
      # a spread of 10 for u_iso's would be enormous for our low temperature
      # test structures if those u_iso's were not constrained
      xs.shake_adp(spread=10, # absolute
                   aniso_spread=0.2) # relative

    self.reparametrisation = constraints.reparametrisation(
      xs, self.constraints, self.connectivity_table,
      temperature=self.t_celsius)
    obs = fo_sq.as_xray_observations()
    ls = least_squares.crystallographic_ls(
      obs,
      self.reparametrisation,
      weighting_scheme=least_squares.mainstream_shelx_weighting())
    self.cycles = self.normal_eqns_solving_method(ls)
    print ("%i %s iterations to recover from shaking"
           % (self.cycles.n_iterations,
              self.cycles))
    if 0:
      from crys3d.qttbx.xray_structure_viewer import display
      display(xray_structure=xs)

    diff = xray.meaningful_site_cart_differences(xs0, xs)
    assert diff.max_absolute() < self.site_refinement_tolerance,\
           self.__class__.__name__

    if self.shall_refine_thermal_displacements:
      delta_u = []
      for sc, sc0 in itertools.izip(xs.scatterers(), xs0.scatterers()):
        if not sc.flags.use_u_aniso() or not sc0.flags.use_u_aniso(): continue
        delta_u.extend(matrix.col(sc.u_star) - matrix.col(sc0.u_star))
      delta_u = flex.double(delta_u)

      assert flex.max_absolute(delta_u) < self.u_star_refinement_tolerance,\
             self.__class__.__name__
コード例 #18
0
ファイル: __init__.py プロジェクト: dials/cctbx
    def __init__(self,
                 xray_structure,
                 obs_,
                 exti=None,
                 connectivity_table=None):
        if exti is None:
            exti = xray.dummy_extinction_correction()
        adopt_init_args(self, locals())
        assert obs_.fo_sq.anomalous_flag()
        assert not (obs_.twin_fractions and obs_.merohedral_components)

        xray_structure = xray_structure.deep_copy_scatterers()
        for sc in xray_structure.scatterers():
            f = xray.scatterer_flags()
            f.set_use_u_aniso(sc.flags.use_u_aniso())
            f.set_use_u_iso(sc.flags.use_u_iso())
            f.set_use_fp_fdp(True)
            sc.flags = f

        twin_fractions = ()
        it = xray.twin_component(sgtbx.rot_mx((-1, 0, 0, 0, -1, 0, 0, 0, -1)),
                                 0.2, True)
        twin_components = (it, )
        obs = observations.customized_copy(obs_, twin_fractions,
                                           twin_components)
        # reparameterisation needs all fractions
        twin_fractions += twin_components
        if connectivity_table is None:
            connectivity_table = smtbx.utils.connectivity_table(xray_structure)
        reparametrisation = constraints.reparametrisation(
            xray_structure, [],
            connectivity_table,
            twin_fractions=twin_fractions,
            extinction=exti)
        normal_eqns = least_squares.crystallographic_ls(obs, reparametrisation)
        cycles = normal_eqns_solving.naive_iterations(normal_eqns,
                                                      n_max_iterations=10,
                                                      gradient_threshold=1e-7,
                                                      step_threshold=1e-4)
        self.flack_x = it.value
        self.sigma_x = math.sqrt(
            normal_eqns.covariance_matrix(
                jacobian_transpose=reparametrisation.
                jacobian_transpose_matching(
                    reparametrisation.mapping_to_grad_fc_independent_scalars))
            [0])
コード例 #19
0
ファイル: tst_masks.py プロジェクト: dials/cctbx
def exercise_least_squares(xray_structure, fo_sq, mask=None):
    from smtbx.refinement import least_squares
    fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
    xs = xray_structure.deep_copy_scatterers()
    if mask is not None:
        f_mask = mask.f_mask()
    else:
        f_mask = None
    connectivity_table = smtbx.utils.connectivity_table(xs)
    reparametrisation = constraints.reparametrisation(
        structure=xs, constraints=[], connectivity_table=connectivity_table)
    obs = fo_sq.as_xray_observations()
    ls = least_squares.crystallographic_ls(obs,
                                           reparametrisation,
                                           f_mask=f_mask,
                                           weighting_scheme="default")
    cycles = normal_eqns_solving.naive_iterations(ls, n_max_iterations=3)
    return xs
コード例 #20
0
    def check_refinement_stability(self):
        xs = self.xray_structure
        xs0 = self.reference_xray_structure = xs.deep_copy_scatterers()
        mi = xs0.build_miller_set(anomalous_flag=True, d_min=0.5)
        fo_sq = mi.structure_factors_from_scatterers(
            xs0, algorithm="direct").f_calc().norm()
        fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))

        self.shake_selection = flex.bool([
            True if sc.element_symbol() == 'O' else False
            for sc in self.xray_structure.scatterers()
        ])
        xs.shake_fps(selection=self.shake_selection)
        xs.shake_fdps(selection=self.shake_selection)

        self.reparametrisation = constraints.reparametrisation(
            xs,
            self.constraints,
            self.connectivity_table,
            temperature=self.t_celsius)
        self.obs = fo_sq.as_xray_observations()
        ls = least_squares.crystallographic_ls(
            self.obs,
            self.reparametrisation,
            weighting_scheme=least_squares.mainstream_shelx_weighting())
        self.cycles = self.normal_eqns_solving_method(ls)
        print("%i %s iterations to recover from shaking" %
              (self.cycles.n_iterations, self.cycles))

        delta_fp = flex.double([
            sc.fp - sc0.fp
            for sc, sc0 in zip(xs.scatterers(), xs0.scatterers())
        ])

        delta_fdp = flex.double([
            sc.fdp - sc0.fdp
            for sc, sc0 in zip(xs.scatterers(), xs0.scatterers())
        ])

        assert flex.max_absolute(delta_fp) < self.fp_refinement_tolerance,\
               self.__class__.__name__

        assert flex.max_absolute(delta_fdp) < self.fdp_refinement_tolerance,\
               self.__class__.__name__
コード例 #21
0
def exercise_u_iso_proportional_to_pivot_u_iso():
  # Test working constraint
  xs = xray.structure(
    crystal_symmetry=crystal.symmetry(
      unit_cell=(),
      space_group_symbol='hall: P 2x 2y'),
    scatterers=flex.xray_scatterer((
      xray.scatterer('C0', u=0.12),
      xray.scatterer('C1'),
      )))
  r = constraints.ext.reparametrisation(xs.unit_cell())
  sc = xs.scatterers()

  u_iso = r.add(constraints.independent_u_iso_parameter, sc[0])
  u_iso_1 = r.add(constraints.u_iso_proportional_to_pivot_u_iso,
                pivot_u_iso=u_iso,
                multiplier=2,
                scatterer=sc[1])
  r.finalise()
  r.linearise()
  assert approx_equal(u_iso_1.value, 0.24, eps=1e-15)

  # Test conflicting constraints
  xs = xray.structure(
    crystal_symmetry=crystal.symmetry(
      unit_cell=(),
      space_group_symbol='hall: P 2x 2y'),
    scatterers=flex.xray_scatterer((
      xray.scatterer('C0', u=0.12),
      xray.scatterer('C1', u=0.21),
      xray.scatterer('C2')
    )))
  with warnings.catch_warnings(record=True) as w:
    warnings.simplefilter("always")
    r = constraints.reparametrisation(
      structure=xs,
      constraints=[constraints.adp.shared_u((0, 2)),
                   constraints.adp.shared_u((1, 2))],
      connectivity_table=smtbx.utils.connectivity_table(xs))
    assert len(w) == 1
    assert w[-1].category == constraints.ConflictingConstraintWarning
    assert w[-1].message.conflicts == set(((2, 'U'),))
コード例 #22
0
ファイル: __init__.py プロジェクト: cctbx/cctbx-playground
  def __init__(self, xray_structure, obs_, exti=None, connectivity_table=None):
    if exti is None:
      exti = xray.dummy_extinction_correction()
    adopt_init_args(self, locals())
    assert obs_.fo_sq.anomalous_flag()
    xray_structure = xray_structure.deep_copy_scatterers()
    flags = xray_structure.scatterer_flags()
    for sc in xray_structure.scatterers():
      f = xray.scatterer_flags()
      f.set_use_u_aniso(sc.flags.use_u_aniso())
      f.set_use_u_iso(sc.flags.use_u_iso())
      f.set_use_fp_fdp(True)
      sc.flags = f

    twin_fractions = obs_.twin_fractions
    twin_components = obs_.merohedral_components
    for tw in twin_fractions: tw.grad = False
    for tc in twin_components: tc.grad = False

    it = xray.twin_component(sgtbx.rot_mx((-1,0,0,0,-1,0,0,0,-1)), 0.2, True)
    twin_components += (it,)
    obs = observations.customized_copy(obs_, twin_fractions, twin_components)
    # reparameterisation needs all fractions
    twin_fractions += twin_components
    if connectivity_table is None:
      connectivity_table = smtbx.utils.connectivity_table(xray_structure)
    reparametrisation = constraints.reparametrisation(
      xray_structure, [], connectivity_table,
      twin_fractions=twin_fractions,
      extinction=exti
    )
    normal_eqns = least_squares.crystallographic_ls(obs,
      reparametrisation)
    cycles = normal_eqns_solving.naive_iterations(
      normal_eqns, n_max_iterations=10,
      gradient_threshold=1e-7,
      step_threshold=1e-4)
    self.flack_x = it.value
    self.sigma_x = math.sqrt(normal_eqns.covariance_matrix(
      jacobian_transpose=reparametrisation.jacobian_transpose_matching(
        reparametrisation.mapping_to_grad_fc_independent_scalars))[0])
コード例 #23
0
    def run(self):
        print("[ %s ]" % self.__class__.__name__)
        self.connectivity_table = smtbx.utils.connectivity_table(
            self.xray_structure)
        for sc in self.xray_structure.scatterers():
            sc.flags.set_grad_site(True)
            if sc.flags.use_u_aniso(): sc.flags.set_grad_u_aniso(True)
            if sc.flags.use_u_iso(): sc.flags.set_grad_u_iso(True)

        self.reparametrisation = constraints.reparametrisation(
            self.xray_structure,
            self.constraints,
            self.connectivity_table,
            temperature=self.t_celsius,
        )

        self.check_reparametrisation_construction()
        self.check_mapping_to_grad_fc()

        # above settings are customised in the following tests
        self.check_refinement_stability()
コード例 #24
0
  def run(self):
    print "[ %s ]" % self.__class__.__name__
    self.connectivity_table = smtbx.utils.connectivity_table(
      self.xray_structure)
    for sc in self.xray_structure.scatterers():
      sc.flags.set_grad_site(True)
      if sc.flags.use_u_aniso(): sc.flags.set_grad_u_aniso(True)
      if sc.flags.use_u_iso(): sc.flags.set_grad_u_iso(True)

    self.reparametrisation = constraints.reparametrisation(
      self.xray_structure,
      self.constraints,
      self.connectivity_table,
      temperature=self.t_celsius,
    )

    self.check_reparametrisation_construction()
    self.check_mapping_to_grad_fc()

    # above settings are customised in the following tests
    self.check_refinement_stability()
コード例 #25
0
    def exercise_ls_cycles(self):
        xs = self.xray_structure.deep_copy_scatterers()
        xs.shake_adp(
        )  # it must happen before the reparamtrisation is constructed
        # because the ADP values are read then and only then.
        connectivity_table = smtbx.utils.connectivity_table(xs)
        reparametrisation = constraints.reparametrisation(
            structure=xs,
            constraints=[],
            connectivity_table=connectivity_table)
        ls = least_squares.crystallographic_ls(
            self.fo_sq.as_xray_observations(),
            reparametrisation,
            weighting_scheme=least_squares.mainstream_shelx_weighting(a=0),
            origin_fixing_restraints_type=oop.null())

        try:
            cycles = normal_eqns_solving.naive_iterations(
                ls, gradient_threshold=1e-12, track_all=True)

            assert approx_equal(ls.scale_factor(), 1, eps=1e-4)
            assert approx_equal(ls.objective(), 0)
            assert cycles.gradient_norm_history[-1] < cycles.gradient_threshold

            for sc0, sc1 in zip(self.xray_structure.scatterers(),
                                xs.scatterers()):
                assert approx_equal(sc0.u_star, sc1.u_star)
        except RuntimeError, err:
            import re
            m = re.search(
                r'^cctbx::adptbx::debye_waller_factor_exp: \s* arg_limit \s+ exceeded'
                '.* arg \s* = \s* ([\d.eE+-]+)', str(err), re.X)
            assert m is not None, eval
            print "Warning: refinement of ADP's diverged"
            print '         argument to debye_waller_factor_exp reached %s' % m.group(
                1)
            print 'Here is the failing structure'
            xs.show_summary()
            xs.show_scatterers()
            raise self.refinement_diverged()
コード例 #26
0
def exercise_u_iso_proportional_to_pivot_u_iso():
    # Test working constraint
    xs = xray.structure(crystal_symmetry=crystal.symmetry(
        unit_cell=(), space_group_symbol='hall: P 2x 2y'),
                        scatterers=flex.xray_scatterer((
                            xray.scatterer('C0', u=0.12),
                            xray.scatterer('C1'),
                        )))
    r = constraints.ext.reparametrisation(xs.unit_cell())
    sc = xs.scatterers()

    u_iso = r.add(constraints.independent_u_iso_parameter, sc[0])
    u_iso_1 = r.add(constraints.u_iso_proportional_to_pivot_u_iso,
                    pivot_u_iso=u_iso,
                    multiplier=2,
                    scatterer=sc[1])
    r.finalise()
    r.linearise()
    assert approx_equal(u_iso_1.value, 0.24, eps=1e-15)

    # Test conflicting constraints
    xs = xray.structure(
        crystal_symmetry=crystal.symmetry(unit_cell=(),
                                          space_group_symbol='hall: P 2x 2y'),
        scatterers=flex.xray_scatterer(
            (xray.scatterer('C0', u=0.12), xray.scatterer('C1', u=0.21),
             xray.scatterer('C2'))))
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        r = constraints.reparametrisation(
            structure=xs,
            constraints=[
                constraints.adp.shared_u((0, 2)),
                constraints.adp.shared_u((1, 2))
            ],
            connectivity_table=smtbx.utils.connectivity_table(xs))
        assert len(w) == 1
        assert w[-1].category == constraints.ConflictingConstraintWarning
        assert w[-1].message.conflicts == set(((2, 'U'), ))
コード例 #27
0
ファイル: tst_masks.py プロジェクト: cctbx/cctbx-playground
def exercise_least_squares(xray_structure, fo_sq, mask=None):
  from smtbx.refinement import least_squares
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(),1.))
  xs = xray_structure.deep_copy_scatterers()
  if mask is not None:
    f_mask = mask.f_mask()
  else:
    f_mask = None
  connectivity_table = smtbx.utils.connectivity_table(xs)
  reparametrisation = constraints.reparametrisation(
    structure=xs,
    constraints=[],
    connectivity_table=connectivity_table)
  obs = fo_sq.as_xray_observations()
  ls = least_squares.crystallographic_ls(
    obs,
    reparametrisation,
    f_mask=f_mask,
    weighting_scheme="default")
  cycles = normal_eqns_solving.naive_iterations(ls,
                                                n_max_iterations=3)
  return xs
コード例 #28
0
  def exercise_ls_cycles(self):
    xs = self.xray_structure.deep_copy_scatterers()
    xs.shake_adp() # it must happen before the reparamtrisation is constructed
                   # because the ADP values are read then and only then.
    connectivity_table = smtbx.utils.connectivity_table(xs)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.mainstream_shelx_weighting(a=0),
      origin_fixing_restraints_type=oop.null())

    try:
      cycles = normal_eqns_solving.naive_iterations(
        ls,
        gradient_threshold=1e-12,
        track_all=True)

      assert approx_equal(ls.scale_factor(), 1, eps=1e-4)
      assert approx_equal(ls.objective(), 0)
      assert cycles.gradient_norm_history[-1] < cycles.gradient_threshold

      for sc0, sc1 in zip(self.xray_structure.scatterers(), xs.scatterers()):
        assert approx_equal(sc0.u_star, sc1.u_star)
    except RuntimeError, err:
      import re
      m = re.search(
        r'^cctbx::adptbx::debye_waller_factor_exp: \s* arg_limit \s+ exceeded'
        '.* arg \s* = \s* ([\d.eE+-]+)', str(err), re.X)
      assert m is not None, eval
      print "Warning: refinement of ADP's diverged"
      print '         argument to debye_waller_factor_exp reached %s' % m.group(1)
      print 'Here is the failing structure'
      xs.show_summary()
      xs.show_scatterers()
      raise self.refinement_diverged()
コード例 #29
0
  def exercise_floating_origin_restraints(self):
    n = self.n_independent_params
    eps_zero_rhs = 1e-6
    connectivity_table = smtbx.utils.connectivity_table(self.xray_structure)
    reparametrisation = constraints.reparametrisation(
      structure=self.xray_structure,
      constraints=[],
      connectivity_table=connectivity_table)
    obs = self.fo_sq.as_xray_observations()
    ls = least_squares.crystallographic_ls(
      obs, reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=oop.null())
    ls.build_up()
    unrestrained_normal_matrix = ls.normal_matrix_packed_u()
    assert len(unrestrained_normal_matrix) == n*(n+1)//2
    ev = eigensystem.real_symmetric(
      unrestrained_normal_matrix.matrix_packed_u_as_symmetric())
    unrestrained_eigenval = ev.values()
    unrestrained_eigenvec = ev.vectors()

    ls = least_squares.crystallographic_ls(
      obs,
      reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.homogeneous_weighting)
    ls.build_up()

    # Let's check that the computed singular directions span the same
    # space as the expected ones
    singular_test = flex.double()
    jac = ls.reparametrisation.jacobian_transpose_matching_grad_fc()
    m = 0
    for s in ls.origin_fixing_restraint.singular_directions:
      assert s.norm() != 0
      singular_test.extend(jac*s)
      m += 1
    for s in self.continuous_origin_shift_basis:
      singular_test.extend(flex.double(s))
      m += 1
    singular_test.reshape(flex.grid(m, n))
    assert self.rank(singular_test) == len(self.continuous_origin_shift_basis)

    assert ls.opposite_of_gradient()\
             .all_approx_equal(0, eps_zero_rhs),\
           list(ls.gradient())
    restrained_normal_matrix = ls.normal_matrix_packed_u()
    assert len(restrained_normal_matrix) == n*(n+1)//2
    ev = eigensystem.real_symmetric(
      restrained_normal_matrix.matrix_packed_u_as_symmetric())
    restrained_eigenval = ev.values()
    restrained_eigenvec = ev.vectors()

    # The eigendecomposition of the normal matrix
    # for the unrestrained problem is:
    #    A = sum_{0 <= i < n-p-1} lambda_i v_i^T v_i
    # where the eigenvalues lambda_i are sorted in decreasing order
    # and p is the dimension of the continous origin shift space.
    # In particular A v_i = 0, n-p <= i < n.
    # In the restrained case, it becomes:
    #    A' = A + sum_{n-p <= i < n} mu v_i^T v_i

    p = len(self.continuous_origin_shift_basis)
    assert approx_equal(restrained_eigenval[p:], unrestrained_eigenval[:-p],
                        eps=1e-12)
    assert unrestrained_eigenval[-p]/unrestrained_eigenval[-p-1] < 1e-12

    if p > 1:
      # eigenvectors are stored by rows
      unrestrained_null_space = unrestrained_eigenvec.matrix_copy_block(
        i_row=n-p, i_column=0,
        n_rows=p, n_columns=n)
      assert self.rank(unrestrained_null_space) == p

      restrained_space = restrained_eigenvec.matrix_copy_block(
        i_row=0, i_column=0,
        n_rows=p, n_columns=n)
      assert self.rank(restrained_space) == p

      singular = flex.double(
        self.continuous_origin_shift_basis)
      assert self.rank(singular) == p

      rank_finder = flex.double(n*3*p)
      rank_finder.resize(flex.grid(3*p, n))
      rank_finder.matrix_paste_block_in_place(unrestrained_null_space,
                                              i_row=0, i_column=0)
      rank_finder.matrix_paste_block_in_place(restrained_space,
                                              i_row=p, i_column=0)
      rank_finder.matrix_paste_block_in_place(singular,
                                              i_row=2*p, i_column=0)
      assert self.rank(rank_finder) == p
    else:
      # this branch handles the case p=1
      # it's necessary to work around a bug in the svd module
      # ( nx1 matrices crashes the code )
      assert approx_equal(
        restrained_eigenvec[0:n].angle(
          unrestrained_eigenvec[-n:]) % math.pi, 0)
      assert approx_equal(
        unrestrained_eigenvec[-n:].angle(
          flex.double(self.continuous_origin_shift_basis[0])) % math.pi, 0)

    # Do the floating origin restraints prevent the structure from floating?
    xs = self.xray_structure.deep_copy_scatterers()
    ls = least_squares.crystallographic_ls(
      obs,
      reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting
    )
    barycentre_0 = xs.sites_frac().mean()
    while True:
      xs.shake_sites_in_place(rms_difference=0.15)
      xs.apply_symmetry_sites()
      barycentre_1 = xs.sites_frac().mean()
      delta = matrix.col(barycentre_1) - matrix.col(barycentre_0)
      moved_far_enough = 0
      for singular in self.continuous_origin_shift_basis:
        e = matrix.col(singular[:3])
        if not approx_equal(delta.dot(e), 0, eps=0.01, out=None):
          moved_far_enough += 1
      if moved_far_enough: break

    # one refinement cycle
    ls.build_up()
    ls.solve()
    shifts = ls.step()

    # That's what floating origin restraints are for!
    # Note that in the presence of special position, that's different
    # from the barycentre not moving along the continuous shift directions.
    # TODO: typeset notes about that subtlety.
    for singular in self.continuous_origin_shift_basis:
      assert approx_equal(shifts.dot(flex.double(singular)), 0, eps=1e-12)
コード例 #30
0
ファイル: tst_lbfgs.py プロジェクト: zhuligs/cctbx_project
def exercise_constrained_lbfgs(xray_structure,
                               constraints_list,
                               t_celsius,
                               d_min=0.5,
                               shake_sites_rmsd=0.5,
                               shake_u_iso_spread=0,
                               shake_u_aniso_spread=0,
                               grad_site=True,
                               grad_u_iso=True,
                               grad_u_aniso=False,
                               grad_occupancy=False,
                               grad_fp_fdp=False,
                               lbfgs_m=5,
                               lbfgs_max_iterations=1000,
                               verbose=0):

    xs = xray_structure
    xray.set_scatterer_grad_flags(scatterers=xs.scatterers(),
                                  site=grad_site,
                                  u_iso=grad_u_iso,
                                  u_aniso=grad_u_aniso,
                                  occupancy=grad_occupancy,
                                  fp=grad_fp_fdp,
                                  fdp=grad_fp_fdp,
                                  tan_u_iso=False,
                                  param=0)

    xs0 = xs.deep_copy_scatterers()
    mi = xs0.build_miller_set(anomalous_flag=False, d_min=d_min)
    fo_sq = mi.structure_factors_from_scatterers(
        xs0, algorithm="direct").f_calc().norm()
    fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
    fo_sq.set_observation_type_xray_intensity()
    y_obs = fo_sq
    #y_obs = fo_sq.f_sq_as_f()
    if grad_site:
        xs.shake_sites_in_place(rms_difference=shake_sites_rmsd)
    if not grad_u_aniso: shake_u_aniso_spread = 0
    if not grad_u_iso: shake_u_iso_spread = 0
    if grad_u_aniso or grad_u_iso:
        xs.shake_adp(spread=shake_u_iso_spread,
                     aniso_spread=shake_u_aniso_spread)
    xs1 = xs.deep_copy_scatterers()

    core_params = scitbx.lbfgs.core_parameters(m=lbfgs_m,
                                               maxfev=100,
                                               xtol=1e-5)

    connectivity_table = smtbx.utils.connectivity_table(xs0)

    if constraints_list is None:
        from smtbx.development import generate_hydrogen_constraints
        constraints_list = generate_hydrogen_constraints(
            structure=xs0, connectivity_table=connectivity_table)

    reparametrisation = constraints.reparametrisation(xs,
                                                      constraints_list,
                                                      connectivity_table,
                                                      temperature=t_celsius)

    lbfgs_termination_params = scitbx.lbfgs.termination_parameters(
        traditional_convergence_test=False,
        drop_convergence_test_max_drop_eps=1.e-20,
        drop_convergence_test_iteration_coefficient=1,
        min_iterations=500,
        max_iterations=lbfgs_max_iterations)

    minimizer = lbfgs(
        target_functor=xray.target_functors.unified_least_squares_residual(
            y_obs),
        xray_structure=xs,
        reparametrisation=reparametrisation,
        structure_factor_algorithm="direct",
        lbfgs_termination_params=lbfgs_termination_params,
        lbfgs_core_params=core_params,
        reference_structure=xs0,
        verbose=verbose)

    if verbose > 0:
        print "Total parameters: ", xs.n_parameters()
        print "Independent parameters: ", reparametrisation.n_independents

        print "Reference model: "
        xs0.show_angles(distance_cutoff=1.5)
        print
        print "Starting model: "
        xs1.show_angles(distance_cutoff=1.5)
        print
        print "Refined model: "
        xs.show_angles(distance_cutoff=1.5)
        print

        print "n_iter, n_fun: ", minimizer.minimizer.iter(
        ), minimizer.minimizer.nfun()

    h_selection = xs.element_selection('H')

    diff = xray.meaningful_site_cart_differences(
        xs0.select(h_selection, negate=True),
        xs.select(h_selection, negate=True))
    #diff = xray.meaningful_site_cart_differences(xs0, xs)
    #assert diff.max_absolute() < 1e-3
    if verbose > 0:
        diff.show()
        print
    assert diff.max_absolute() < 2e-2, diff.max_absolute()

    diff = xray.meaningful_site_cart_differences(xs0, xs)
    if verbose > 0:
        diff.show()
        print
    # XXX why does this tolerance have to be so high?
    assert diff.max_absolute() < 0.5, diff.max_absolute()
コード例 #31
0
ファイル: tst_lbfgs.py プロジェクト: cctbx/cctbx-playground
def exercise_constrained_lbfgs(xray_structure,
                               constraints_list,
                               t_celsius,
                               d_min=0.5,
                               shake_sites_rmsd=0.5,
                               shake_u_iso_spread=0,
                               shake_u_aniso_spread=0,
                               grad_site=True,
                               grad_u_iso=True,
                               grad_u_aniso=False,
                               grad_occupancy=False,
                               grad_fp_fdp=False,
                               lbfgs_m=5,
                               lbfgs_max_iterations=1000,
                               verbose=0):

  xs = xray_structure
  xray.set_scatterer_grad_flags(scatterers=xs.scatterers(),
                                site=grad_site,
                                u_iso=grad_u_iso,
                                u_aniso=grad_u_aniso,
                                occupancy=grad_occupancy,
                                fp=grad_fp_fdp,
                                fdp=grad_fp_fdp,
                                tan_u_iso=False,
                                param=0)

  xs0 = xs.deep_copy_scatterers()
  mi = xs0.build_miller_set(anomalous_flag=False, d_min=d_min)
  fo_sq = mi.structure_factors_from_scatterers(
    xs0, algorithm="direct").f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
  fo_sq.set_observation_type_xray_intensity()
  y_obs = fo_sq
  #y_obs = fo_sq.f_sq_as_f()
  if grad_site:
    xs.shake_sites_in_place(rms_difference=shake_sites_rmsd)
  if not grad_u_aniso: shake_u_aniso_spread = 0
  if not grad_u_iso: shake_u_iso_spread = 0
  if grad_u_aniso or grad_u_iso:
    xs.shake_adp(spread=shake_u_iso_spread, aniso_spread=shake_u_aniso_spread)
  xs1 = xs.deep_copy_scatterers()

  core_params = scitbx.lbfgs.core_parameters(m=lbfgs_m, maxfev=100, xtol=1e-5)

  connectivity_table = smtbx.utils.connectivity_table(xs0)

  if constraints_list is None:
    from smtbx.development import generate_hydrogen_constraints
    constraints_list = generate_hydrogen_constraints(
      structure=xs0, connectivity_table=connectivity_table)

  reparametrisation = constraints.reparametrisation(
    xs,
    constraints_list,
    connectivity_table,
    temperature=t_celsius)

  lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
    traditional_convergence_test=False,
    drop_convergence_test_max_drop_eps=1.e-20,
    drop_convergence_test_iteration_coefficient=1,
    min_iterations=500,
    max_iterations=lbfgs_max_iterations)

  minimizer = lbfgs(
    target_functor=xray.target_functors.unified_least_squares_residual(y_obs),
    xray_structure=xs,
    reparametrisation=reparametrisation,
    structure_factor_algorithm="direct",
    lbfgs_termination_params=lbfgs_termination_params,
    lbfgs_core_params=core_params,
    reference_structure=xs0,
    verbose=verbose)

  if verbose > 0:
    print "Total parameters: ", xs.n_parameters()
    print "Independent parameters: ", reparametrisation.n_independents

    print "Reference model: "
    xs0.show_angles(distance_cutoff=1.5)
    print
    print "Starting model: "
    xs1.show_angles(distance_cutoff=1.5)
    print
    print "Refined model: "
    xs.show_angles(distance_cutoff=1.5)
    print

    print "n_iter, n_fun: ", minimizer.minimizer.iter(), minimizer.minimizer.nfun()

  h_selection = xs.element_selection('H')

  diff = xray.meaningful_site_cart_differences(
    xs0.select(h_selection, negate=True),
    xs.select(h_selection, negate=True))
  #diff = xray.meaningful_site_cart_differences(xs0, xs)
  #assert diff.max_absolute() < 1e-3
  if verbose > 0:
    diff.show()
    print
  assert diff.max_absolute() < 2e-2, diff.max_absolute()

  diff = xray.meaningful_site_cart_differences(xs0, xs)
  if verbose > 0:
    diff.show()
    print
  # XXX why does this tolerance have to be so high?
  assert diff.max_absolute() < 0.5, diff.max_absolute()
コード例 #32
0
def exercise_floating_origin_dynamic_weighting(verbose=False):
  from cctbx import covariance
  import scitbx.math

  worst_condition_number_acceptable = 10

  # light elements only
  xs0 = random_structure.xray_structure(elements=['C', 'C', 'C', 'O', 'N'],
                                        use_u_aniso=True)
  msg = "light elements in %s ..." % (
    xs0.space_group_info().type().hall_symbol())
  if verbose:
    print(msg, end=' ')
  fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
  xs = xs0.deep_copy_scatterers()
  xs.shake_adp()
  xs.shake_sites_in_place(rms_difference=0.1)
  for sc in xs.scatterers():
    sc.flags.set_grad_site(True).set_grad_u_aniso(True)
  ls = least_squares.crystallographic_ls(
    fo_sq.as_xray_observations(),
    constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=smtbx.utils.connectivity_table(xs)),
    weighting_scheme=least_squares.unit_weighting(),
    origin_fixing_restraints_type=
    origin_fixing_restraints.atomic_number_weighting)
  ls.build_up()
  lambdas = eigensystem.real_symmetric(
    ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
  # assert the restrained L.S. problem is not too ill-conditionned
  cond = math.log10(lambdas[0]/lambdas[-1])
  if verbose:
    print("normal matrix condition: %.1f" % cond)
  assert cond < worst_condition_number_acceptable, msg

  # one heavy element
  xs0 = random_structure.xray_structure(
    space_group_info=sgtbx.space_group_info('hall: P 2yb'),
    elements=['Zn', 'C', 'C', 'C', 'O', 'N'],
    use_u_aniso=True)
  msg = "one heavy element + light elements (synthetic data) in %s ..." % (
    xs0.space_group_info().type().hall_symbol())
  if verbose:
    print(msg, end=' ')
  fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
  xs = xs0.deep_copy_scatterers()
  xs.shake_adp()
  xs.shake_sites_in_place(rms_difference=0.1)
  for sc in xs.scatterers():
    sc.flags.set_grad_site(True).set_grad_u_aniso(True)
  ls = least_squares.crystallographic_ls(
    fo_sq.as_xray_observations(),
    constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=smtbx.utils.connectivity_table(xs)),
    weighting_scheme=least_squares.mainstream_shelx_weighting(),
    origin_fixing_restraints_type=
    origin_fixing_restraints.atomic_number_weighting)
  ls.build_up()
  lambdas = eigensystem.real_symmetric(
    ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
  # assert the restrained L.S. problem is not too ill-conditionned
  cond = math.log10(lambdas[0]/lambdas[-1])
  if verbose:
    print("normal matrix condition: %.1f" % cond)
  assert cond < worst_condition_number_acceptable, msg

  # are esd's for x,y,z coordinates of the same order of magnitude?
  var_cart = covariance.orthogonalize_covariance_matrix(
    ls.covariance_matrix(),
    xs.unit_cell(),
    xs.parameter_map())
  var_site_cart = covariance.extract_covariance_matrix_for_sites(
      flex.size_t_range(len(xs.scatterers())),
      var_cart,
      xs.parameter_map())
  site_esds = var_site_cart.matrix_packed_u_diagonal()
  indicators = flex.double()
  for i in xrange(0, len(site_esds), 3):
    stats = scitbx.math.basic_statistics(site_esds[i:i+3])
    indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
  assert indicators.all_lt(2)

  # especially troublesome structure with one heavy element
  # (contributed by Jonathan Coome)
  xs0 = xray.structure(
    crystal_symmetry=crystal.symmetry(
      unit_cell=(8.4519, 8.4632, 18.7887, 90, 96.921, 90),
      space_group_symbol="hall: P 2yb"),
    scatterers=flex.xray_scatterer([
      xray.scatterer( #0
                      label="ZN1",
                      site=(-0.736683, -0.313978, -0.246902),
                      u=(0.000302, 0.000323, 0.000054,
                         0.000011, 0.000015, -0.000004)),
      xray.scatterer( #1
                      label="N3B",
                      site=(-0.721014, -0.313583, -0.134277),
                      u=(0.000268, 0.000237, 0.000055,
                         -0.000027, 0.000005, 0.000006)),
      xray.scatterer( #2
                      label="N3A",
                      site=(-0.733619, -0.290423, -0.357921),
                      u=(0.000229, 0.000313, 0.000053,
                         0.000022, 0.000018, -0.000018)),
      xray.scatterer( #3
                      label="C9B",
                      site=(-1.101537, -0.120157, -0.138063),
                      u=(0.000315, 0.000345, 0.000103,
                         0.000050, 0.000055, -0.000017)),
    xray.scatterer( #4
                    label="N5B",
                    site=(-0.962032, -0.220345, -0.222045),
                    u=(0.000274, 0.000392, 0.000060,
                       -0.000011, -0.000001, -0.000002)),
    xray.scatterer( #5
                    label="N1B",
                    site=(-0.498153, -0.402742, -0.208698),
                    u=(0.000252, 0.000306, 0.000063,
                       0.000000, 0.000007, 0.000018)),
    xray.scatterer( #6
                    label="C3B",
                    site=(-0.322492, -0.472610, -0.114594),
                    u=(0.000302, 0.000331, 0.000085,
                       0.000016, -0.000013, 0.000037)),
    xray.scatterer( #7
                    label="C4B",
                    site=(-0.591851, -0.368163, -0.094677),
                    u=(0.000262, 0.000255, 0.000073,
                       -0.000034, 0.000027, -0.000004)),
    xray.scatterer( #8
                    label="N4B",
                    site=(-0.969383, -0.204624, -0.150014),
                    u=(0.000279, 0.000259, 0.000070,
                       -0.000009, 0.000039, 0.000000)),
    xray.scatterer( #9
                    label="N2B",
                    site=(-0.470538, -0.414572, -0.135526),
                    u=(0.000277, 0.000282, 0.000065,
                       0.000003, 0.000021, -0.000006)),
    xray.scatterer( #10
                    label="C8A",
                    site=(-0.679889, -0.158646, -0.385629),
                    u=(0.000209, 0.000290, 0.000078,
                       0.000060, 0.000006, 0.000016)),
    xray.scatterer( #11
                    label="N5A",
                    site=(-0.649210, -0.075518, -0.263412),
                    u=(0.000307, 0.000335, 0.000057,
                       -0.000002, 0.000016, -0.000012)),
    xray.scatterer( #12
                    label="C6B",
                    site=(-0.708620, -0.325965, 0.011657),
                    u=(0.000503, 0.000318, 0.000053,
                       -0.000058, 0.000032, -0.000019)),
    xray.scatterer( #13
                    label="C10B",
                    site=(-1.179332, -0.083184, -0.202815),
                    u=(0.000280, 0.000424, 0.000136,
                       0.000094, 0.000006, 0.000013)),
    xray.scatterer( #14
                    label="N1A",
                    site=(-0.838363, -0.532191, -0.293213),
                    u=(0.000312, 0.000323, 0.000060,
                       0.000018, 0.000011, -0.000008)),
    xray.scatterer( #15
                    label="C3A",
                    site=(-0.915414, -0.671031, -0.393826),
                    u=(0.000319, 0.000384, 0.000078,
                       -0.000052, -0.000001, -0.000020)),
    xray.scatterer( #16
                    label="C1A",
                    site=(-0.907466, -0.665419, -0.276011),
                    u=(0.000371, 0.000315, 0.000079,
                       0.000006, 0.000036, 0.000033)),
    xray.scatterer( #17
                    label="C1B",
                    site=(-0.365085, -0.452753, -0.231927),
                    u=(0.000321, 0.000253, 0.000087,
                       -0.000024, 0.000047, -0.000034)),
    xray.scatterer( #18
                    label="C11A",
                    site=(-0.598622, 0.053343, -0.227354),
                    u=(0.000265, 0.000409, 0.000084,
                       0.000088, -0.000018, -0.000030)),
    xray.scatterer( #19
                    label="C2A",
                    site=(-0.958694, -0.755645, -0.337016),
                    u=(0.000394, 0.000350, 0.000106,
                       -0.000057, 0.000027, -0.000005)),
    xray.scatterer( #20
                    label="C4A",
                    site=(-0.784860, -0.407601, -0.402050),
                    u=(0.000238, 0.000296, 0.000064,
                       0.000002, 0.000011, -0.000016)),
    xray.scatterer( #21
                    label="C5A",
                    site=(-0.784185, -0.399716, -0.475491),
                    u=(0.000310, 0.000364, 0.000062,
                       0.000044, -0.000011, -0.000017)),
    xray.scatterer( #22
                    label="N4A",
                    site=(-0.630284, -0.043981, -0.333143),
                    u=(0.000290, 0.000275, 0.000074,
                       0.000021, 0.000027, 0.000013)),
    xray.scatterer( #23
                    label="C10A",
                    site=(-0.545465, 0.166922, -0.272829),
                    u=(0.000369, 0.000253, 0.000117,
                       0.000015, -0.000002, -0.000008)),
    xray.scatterer( #24
                    label="C9A",
                    site=(-0.567548, 0.102272, -0.339923),
                    u=(0.000346, 0.000335, 0.000103,
                       -0.000016, 0.000037, 0.000023)),
    xray.scatterer( #25
                    label="C11B",
                    site=(-1.089943, -0.146930, -0.253779),
                    u=(0.000262, 0.000422, 0.000102,
                       -0.000018, -0.000002, 0.000029)),
    xray.scatterer( #26
                    label="N2A",
                    site=(-0.843385, -0.537780, -0.366515),
                    u=(0.000273, 0.000309, 0.000055,
                       -0.000012, -0.000005, -0.000018)),
    xray.scatterer( #27
                    label="C7A",
                    site=(-0.674021, -0.136086, -0.457790),
                    u=(0.000362, 0.000378, 0.000074,
                       0.000043, 0.000034, 0.000016)),
    xray.scatterer( #28
                    label="C8B",
                    site=(-0.843625, -0.264182, -0.102023),
                    u=(0.000264, 0.000275, 0.000072,
                       -0.000025, 0.000019, -0.000005)),
    xray.scatterer( #29
                    label="C6A",
                    site=(-0.726731, -0.261702, -0.502366),
                    u=(0.000339, 0.000472, 0.000064,
                       0.000062, -0.000003, 0.000028)),
    xray.scatterer( #30
                    label="C5B",
                    site=(-0.577197, -0.376753, -0.020800),
                    u=(0.000349, 0.000353, 0.000066,
                       -0.000082, -0.000022, 0.000014)),
    xray.scatterer( #31
                    label="C2B",
                    site=(-0.252088, -0.497338, -0.175057),
                    u=(0.000251, 0.000342, 0.000119,
                       0.000020, 0.000034, -0.000018)),
    xray.scatterer( #32
                    label="C7B",
                    site=(-0.843956, -0.268811, -0.028080),
                    u=(0.000344, 0.000377, 0.000078,
                       -0.000029, 0.000059, -0.000007)),
    xray.scatterer( #33
                    label="F4B",
                    site=(-0.680814, -0.696808, -0.115056),
                    u=(0.000670, 0.000408, 0.000109,
                       -0.000099, 0.000139, -0.000031)),
    xray.scatterer( #34
                    label="F1B",
                    site=(-0.780326, -0.921249, -0.073962),
                    u=(0.000687, 0.000357, 0.000128,
                       -0.000152, -0.000011, 0.000021)),
    xray.scatterer( #35
                    label="B1B",
                    site=(-0.795220, -0.758128, -0.075955),
                    u=(0.000413, 0.000418, 0.000075,
                       0.000054, 0.000045, 0.000023)),
    xray.scatterer( #36
                    label="F2B",
                    site=(-0.945140, -0.714626, -0.105820),
                    u=(0.000584, 0.001371, 0.000108,
                       0.000420, 0.000067, 0.000134)),
    xray.scatterer( #37
                    label="F3B",
                    site=(-0.768914, -0.701660, -0.005161),
                    u=(0.000678, 0.000544, 0.000079,
                       -0.000000, 0.000090, -0.000021)),
    xray.scatterer( #38
                    label="F1A",
                    site=(-0.109283, -0.252334, -0.429288),
                    u=(0.000427, 0.001704, 0.000125,
                       0.000407, 0.000041, 0.000035)),
    xray.scatterer( #39
                    label="F4A",
                    site=(-0.341552, -0.262864, -0.502023),
                    u=(0.000640, 0.000557, 0.000081,
                       -0.000074, 0.000042, -0.000052)),
    xray.scatterer( #40
                    label="F3A",
                    site=(-0.324533, -0.142292, -0.393215),
                    u=(0.000471, 0.001203, 0.000134,
                       0.000333, -0.000057, -0.000220)),
    xray.scatterer( #41
                    label="F2A",
                    site=(-0.312838, -0.405405, -0.400231),
                    u=(0.002822, 0.000831, 0.000092,
                       -0.000648, 0.000115, 0.000027)),
    xray.scatterer( #42
                    label="B1A",
                    site=(-0.271589, -0.268874, -0.430724),
                    u=(0.000643, 0.000443, 0.000079,
                       0.000040, 0.000052, -0.000034)),
    xray.scatterer( #43
                    label="H5B",
                    site=(-0.475808, -0.413802, 0.004402),
                    u=0.005270),
    xray.scatterer( #44
                    label="H6B",
                    site=(-0.699519, -0.326233, 0.062781),
                    u=0.019940),
    xray.scatterer( #45
                    label="H3B",
                    site=(-0.283410, -0.484757, -0.063922),
                    u=0.029990),
    xray.scatterer( #46
                    label="H1B",
                    site=(-0.357103, -0.451819, -0.284911),
                    u=0.031070),
    xray.scatterer( #47
                    label="H10A",
                    site=(-0.495517, 0.268296, -0.256187),
                    u=0.027610),
    xray.scatterer( #48
                    label="H2B",
                    site=(-0.147129, -0.535141, -0.174699),
                    u=0.017930),
    xray.scatterer( #49
                    label="H7A",
                    site=(-0.643658, -0.031387, -0.475357),
                    u=0.020200),
    xray.scatterer( #50
                    label="H1A",
                    site=(-0.912757, -0.691043, -0.227554),
                    u=0.033320),
    xray.scatterer( #51
                    label="H7B",
                    site=(-0.933670, -0.241189, -0.010263),
                    u=0.021310),
    xray.scatterer( #52
                    label="H11B",
                    site=(-1.107736, -0.155470, -0.311996),
                    u=0.041500),
    xray.scatterer( #53
                    label="H9A",
                    site=(-0.539908, 0.139753, -0.382281),
                    u=0.007130),
    xray.scatterer( #54
                    label="H10B",
                    site=(-1.265944, -0.029610, -0.212398),
                    u=0.030910),
    xray.scatterer( #55
                    label="H3A",
                    site=(-0.934728, -0.691149, -0.450551),
                    u=0.038950),
    xray.scatterer( #56
                    label="H5A",
                    site=(-0.833654, -0.487479, -0.508239),
                    u=0.031150),
    xray.scatterer( #57
                    label="H6A",
                    site=(-0.742871, -0.242269, -0.558157),
                    u=0.050490),
    xray.scatterer( #58
                    label="H9B",
                    site=(-1.120150, -0.093752, -0.090706),
                    u=0.039310),
    xray.scatterer( #59
                    label="H11A",
                    site=(-0.593074, 0.054973, -0.180370),
                    u=0.055810),
    xray.scatterer( #60
                    label="H2A",
                    site=(-0.999576, -0.842158, -0.340837),
                    u=0.057030)
    ]))
  fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
  for hydrogen_flag in (True, False):
    xs = xs0.deep_copy_scatterers()
    if not hydrogen_flag:
      xs.select_inplace(~xs.element_selection('H'))
    xs.shake_adp()
    xs.shake_sites_in_place(rms_difference=0.1)
    for sc in xs.scatterers():
      sc.flags.set_grad_site(True).set_grad_u_aniso(False)
    ls = least_squares.crystallographic_ls(
      fo_sq.as_xray_observations(),
      constraints.reparametrisation(
        structure=xs,
        constraints=[],
        connectivity_table=smtbx.utils.connectivity_table(xs)),
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    ls.build_up()
    lambdas = eigensystem.real_symmetric(
      ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
    # assert the restrained L.S. problem is not too ill-conditionned
    cond = math.log10(lambdas[0]/lambdas[-1])
    msg = ("one heavy element + light elements (real data) %s Hydrogens: %.1f"
           % (['without', 'with'][hydrogen_flag], cond))
    if verbose: print(msg)
    assert cond < worst_condition_number_acceptable, msg


    # are esd's for x,y,z coordinates of the same order of magnitude?
    var_cart = covariance.orthogonalize_covariance_matrix(
      ls.covariance_matrix(),
      xs.unit_cell(),
      xs.parameter_map())
    var_site_cart = covariance.extract_covariance_matrix_for_sites(
        flex.size_t_range(len(xs.scatterers())),
        var_cart,
        xs.parameter_map())
    site_esds = var_site_cart.matrix_packed_u_diagonal()
    indicators = flex.double()
    for i in xrange(0, len(site_esds), 3):
      stats = scitbx.math.basic_statistics(site_esds[i:i+3])
      indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
    assert indicators.all_lt(1)
コード例 #33
0
  def exercise(self):
    xs0 = self.structure
    xs = xs0.deep_copy_scatterers()
    k1, s1, li1, o1, o2 = xs.scatterers()
    self.shake_point_group_3(k1)
    self.shake_point_group_3(s1)
    self.shake_point_group_3(li1)
    self.shake_point_group_3(o1)
    o2.site = tuple(
      [ x*(1 + random.uniform(-self.delta_site, self.delta_site))
        for x in o2.site])
    o2.u_star = tuple(
      [ u*(1 + random.uniform(-self.delta_u_star, self.delta_u_star))
        for u in o2.u_star])

    for sc in xs.scatterers():
      sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
      sc.flags.set_grad_site(True).set_grad_u_aniso(True)
      connectivity_table = smtbx.utils.connectivity_table(xs)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    cycles = normal_eqns_solving.levenberg_marquardt_iterations(
      ls,
      gradient_threshold=1e-12,
      step_threshold=1e-7,
      track_all=True)

    ## Test whether refinement brought back the shaked structure to its
    ## original state
    match = emma.model_matches(xs0.as_emma_model(),
                               xs.as_emma_model()).refined_matches[0]
    assert match.rt.r == matrix.identity(3)
    assert not match.singles1 and not match.singles2
    assert match.rms < 1e-6

    delta_u_carts= (   xs.scatterers().extract_u_cart(xs.unit_cell())
                    - xs0.scatterers().extract_u_cart(xs.unit_cell())).norms()
    assert flex.abs(delta_u_carts) < 1e-6

    assert approx_equal(ls.scale_factor(), 1, eps=1e-4)

    ## Test covariance matrix
    jac_tr = reparametrisation.jacobian_transpose_matching_grad_fc()
    cov = ls.covariance_matrix(
      jacobian_transpose=jac_tr, normalised_by_goof=False)\
        .matrix_packed_u_as_symmetric()
    m, n = cov.accessor().focus()
    # x,y for point group 3 sites are fixed: no variance or correlation
    for i in (0, 9, 18, 27,):
      assert cov.matrix_copy_block(i, 0, 2, n) == 0

    # u_star coefficients u13 and u23 for point group 3 sites are fixed
    # to 0: again no variance or correlation with any other param
    for i in (7, 16, 25, 34,):
      assert cov.matrix_copy_block(i, 0, 2, n).as_1d()\
             .all_approx_equal(0., 1e-20)

    # u_star coefficients u11, u22 and u12 for point group 3 sites
    # are totally correlated, with variances in ratios 1:1:1/2
    for i in (3, 12, 21, 30,):
      assert cov[i, i] != 0
      assert approx_equal(cov[i, i], cov[i+1, i+1], eps=1e-15)
      assert approx_equal(cov[i, i+1]/cov[i, i], 1, eps=1e-12)
      assert approx_equal(cov[i, i+3]/cov[i, i], 0.5, eps=1e-12)
コード例 #34
0
  def exercise(self):
    xs0 = self.structure
    xs = xs0.deep_copy_scatterers()
    xs.shake_sites_in_place(rms_difference=0.15)
    xs.shake_adp()

    for sc in xs.scatterers():
      sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
      sc.flags.set_grad_site(True).set_grad_u_aniso(True)
    connectivity_table = smtbx.utils.connectivity_table(xs)
    if self.twin_fractions[0] < 0.5:
      twin_fractions = self.twin_fractions.deep_copy() + 0.1
    else:
      twin_fractions = self.twin_fractions.deep_copy() - 0.1
    twin_components = tuple(
      [xray.twin_component(law, fraction, grad=True)
       for law, fraction in zip(self.twin_laws, twin_fractions)])
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table,
      twin_fractions=twin_components)
    obs = self.fo_sq.as_xray_observations(twin_components=twin_components)
    normal_eqns = least_squares.crystallographic_ls(
      obs, reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)
    cycles = normal_eqns_solving.naive_iterations(
      normal_eqns,
      n_max_iterations=10,
      track_all=True)
    assert approx_equal(
      [twin.value for twin in normal_eqns.twin_fractions],
      self.twin_fractions, eps=1e-2)
    assert approx_equal(normal_eqns.objective(), 0, eps=1e-5)
    assert normal_eqns.n_parameters == 64
    # now with fixed twin fraction
    xs.shake_sites_in_place(rms_difference=0.15)
    xs.shake_adp()
    twin_components = tuple(
      [xray.twin_component(law, fraction, grad=False)
       for law, fraction in zip(self.twin_laws, twin_fractions)])
    #change the twin_components of the observations...
    obs = observations.customized_copy(obs, twin_components=twin_components)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table,
      twin_fractions=twin_components)
    normal_eqns = least_squares.crystallographic_ls(
      obs, reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)
    cycles = normal_eqns_solving.naive_iterations(
      normal_eqns,
      n_max_iterations=10,
      track_all=True)
    assert approx_equal(
      [twin.value for twin in normal_eqns.twin_fractions],
      twin_fractions)
    assert normal_eqns.objective() != 0 # since the twin fraction is not refined
    assert normal_eqns.n_parameters == 63
コード例 #35
0
  def exercise(self):
    xs0 = self.structure
    xs = xs0.deep_copy_scatterers()
    k1, s1, li1, o1, o2 = xs.scatterers()
    self.shake_point_group_3(k1)
    self.shake_point_group_3(s1)
    self.shake_point_group_3(li1)
    self.shake_point_group_3(o1)
    o2.site = tuple(
      [ x*(1 + random.uniform(-self.delta_site, self.delta_site))
        for x in o2.site])
    o2.u_star = tuple(
      [ u*(1 + random.uniform(-self.delta_u_star, self.delta_u_star))
        for u in o2.u_star])

    for sc in xs.scatterers():
      sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
      sc.flags.set_grad_site(True).set_grad_u_aniso(True)
      connectivity_table = smtbx.utils.connectivity_table(xs)
    reparametrisation = constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=connectivity_table)
    ls = least_squares.crystallographic_ls(
      self.fo_sq.as_xray_observations(), reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    cycles = normal_eqns_solving.naive_iterations(
      ls,
      gradient_threshold=1e-6,
      step_threshold=1e-6,
      track_all=True)

    ## Test whether refinement brought back the shaked structure to its
    ## original state
    match = emma.model_matches(xs0.as_emma_model(),
                               xs.as_emma_model()).refined_matches[0]
    assert match.rt.r == matrix.identity(3)
    assert not match.singles1 and not match.singles2
    assert match.rms < 1e-6

    delta_u_carts= (   xs.scatterers().extract_u_cart(xs.unit_cell())
                    - xs0.scatterers().extract_u_cart(xs.unit_cell())).norms()
    assert flex.abs(delta_u_carts) < 1e-6

    assert approx_equal(ls.scale_factor(), 1, eps=1e-4)

    ## Test covariance matrix
    jac_tr = reparametrisation.jacobian_transpose_matching_grad_fc()
    cov = ls.covariance_matrix(
      jacobian_transpose=jac_tr, normalised_by_goof=False)\
        .matrix_packed_u_as_symmetric()
    m, n = cov.accessor().focus()
    # x,y for point group 3 sites are fixed: no variance or correlation
    for i in (0, 9, 18, 27,):
      assert cov.matrix_copy_block(i, 0, 2, n) == 0

    # u_star coefficients u13 and u23 for point group 3 sites are fixed
    # to 0: again no variance or correlation with any other param
    for i in (7, 16, 25, 34,):
      assert cov.matrix_copy_block(i, 0, 2, n).as_1d()\
             .all_approx_equal(0., 1e-20)

    # u_star coefficients u11, u22 and u12 for point group 3 sites
    # are totally correlated, with variances in ratios 1:1:1/2
    for i in (3, 12, 21, 30,):
      assert cov[i, i] != 0
      assert approx_equal(cov[i, i], cov[i+1, i+1], eps=1e-15)
      assert approx_equal(cov[i, i+1]/cov[i, i], 1, eps=1e-12)
      assert approx_equal(cov[i, i+3]/cov[i, i], 0.5, eps=1e-12)
コード例 #36
0
  def exercise_floating_origin_restraints(self):
    n = self.n_independent_params
    eps_zero_rhs = 1e-6
    connectivity_table = smtbx.utils.connectivity_table(self.xray_structure)
    reparametrisation = constraints.reparametrisation(
      structure=self.xray_structure,
      constraints=[],
      connectivity_table=connectivity_table)
    obs = self.fo_sq.as_xray_observations()
    ls = least_squares.crystallographic_ls(
      obs, reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=oop.null())
    ls.build_up()
    unrestrained_normal_matrix = ls.normal_matrix_packed_u()
    assert len(unrestrained_normal_matrix) == n*(n+1)//2
    ev = eigensystem.real_symmetric(
      unrestrained_normal_matrix.matrix_packed_u_as_symmetric())
    unrestrained_eigenval = ev.values()
    unrestrained_eigenvec = ev.vectors()

    ls = least_squares.crystallographic_ls(
      obs,
      reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.homogeneous_weighting)
    ls.build_up()

    # Let's check that the computed singular directions span the same
    # space as the expected ones
    singular_test = flex.double()
    jac = ls.reparametrisation.jacobian_transpose_matching_grad_fc()
    m = 0
    for s in ls.origin_fixing_restraint.singular_directions:
      assert s.norm() != 0
      singular_test.extend(jac*s)
      m += 1
    for s in self.continuous_origin_shift_basis:
      singular_test.extend(flex.double(s))
      m += 1
    singular_test.reshape(flex.grid(m, n))
    assert self.rank(singular_test) == len(self.continuous_origin_shift_basis)

    assert ls.opposite_of_gradient()\
             .all_approx_equal(0, eps_zero_rhs),\
           list(ls.gradient())
    restrained_normal_matrix = ls.normal_matrix_packed_u()
    assert len(restrained_normal_matrix) == n*(n+1)//2
    ev = eigensystem.real_symmetric(
      restrained_normal_matrix.matrix_packed_u_as_symmetric())
    restrained_eigenval = ev.values()
    restrained_eigenvec = ev.vectors()

    # The eigendecomposition of the normal matrix
    # for the unrestrained problem is:
    #    A = sum_{0 <= i < n-p-1} lambda_i v_i^T v_i
    # where the eigenvalues lambda_i are sorted in decreasing order
    # and p is the dimension of the continous origin shift space.
    # In particular A v_i = 0, n-p <= i < n.
    # In the restrained case, it becomes:
    #    A' = A + sum_{n-p <= i < n} mu v_i^T v_i

    p = len(self.continuous_origin_shift_basis)
    assert approx_equal(restrained_eigenval[p:], unrestrained_eigenval[:-p],
                        eps=1e-12)
    assert unrestrained_eigenval[-p]/unrestrained_eigenval[-p-1] < 1e-12

    if p > 1:
      # eigenvectors are stored by rows
      unrestrained_null_space = unrestrained_eigenvec.matrix_copy_block(
        i_row=n-p, i_column=0,
        n_rows=p, n_columns=n)
      assert self.rank(unrestrained_null_space) == p

      restrained_space = restrained_eigenvec.matrix_copy_block(
        i_row=0, i_column=0,
        n_rows=p, n_columns=n)
      assert self.rank(restrained_space) == p

      singular = flex.double(
        self.continuous_origin_shift_basis)
      assert self.rank(singular) == p

      rank_finder = flex.double(n*3*p)
      rank_finder.resize(flex.grid(3*p, n))
      rank_finder.matrix_paste_block_in_place(unrestrained_null_space,
                                              i_row=0, i_column=0)
      rank_finder.matrix_paste_block_in_place(restrained_space,
                                              i_row=p, i_column=0)
      rank_finder.matrix_paste_block_in_place(singular,
                                              i_row=2*p, i_column=0)
      assert self.rank(rank_finder) == p
    else:
      # this branch handles the case p=1
      # it's necessary to work around a bug in the svd module
      # ( nx1 matrices crashes the code )
      assert approx_equal(
        restrained_eigenvec[0:n].angle(
          unrestrained_eigenvec[-n:]) % math.pi, 0)
      assert approx_equal(
        unrestrained_eigenvec[-n:].angle(
          flex.double(self.continuous_origin_shift_basis[0])) % math.pi, 0)

    # Do the floating origin restraints prevent the structure from floating?
    xs = self.xray_structure.deep_copy_scatterers()
    ls = least_squares.crystallographic_ls(
      obs,
      reparametrisation,
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting
    )
    barycentre_0 = xs.sites_frac().mean()
    while 1:
      xs.shake_sites_in_place(rms_difference=0.15)
      xs.apply_symmetry_sites()
      barycentre_1 = xs.sites_frac().mean()
      delta = matrix.col(barycentre_1) - matrix.col(barycentre_0)
      moved_far_enough = 0
      for singular in self.continuous_origin_shift_basis:
        e = matrix.col(singular[:3])
        if not approx_equal(delta.dot(e), 0, eps=0.01, out=None):
          moved_far_enough += 1
      if moved_far_enough: break

    # one refinement cycle
    ls.build_up()
    ls.solve()
    shifts = ls.step()

    # That's what floating origin restraints are for!
    # Note that in the presence of special position, that's different
    # from the barycentre not moving along the continuous shift directions.
    # TODO: typeset notes about that subtlety.
    for singular in self.continuous_origin_shift_basis:
      assert approx_equal(shifts.dot(flex.double(singular)), 0, eps=1e-12)
コード例 #37
0
def exercise_floating_origin_dynamic_weighting(verbose=False):
  from cctbx import covariance
  import scitbx.math

  worst_condition_number_acceptable = 10

  # light elements only
  xs0 = random_structure.xray_structure(elements=['C', 'C', 'C', 'O', 'N'],
                                        use_u_aniso=True)
  fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
  xs = xs0.deep_copy_scatterers()
  xs.shake_adp()
  xs.shake_sites_in_place(rms_difference=0.1)
  for sc in xs.scatterers():
    sc.flags.set_grad_site(True).set_grad_u_aniso(True)
  ls = least_squares.crystallographic_ls(
    fo_sq.as_xray_observations(),
    constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=smtbx.utils.connectivity_table(xs)),
    weighting_scheme=least_squares.unit_weighting(),
    origin_fixing_restraints_type=
    origin_fixing_restraints.atomic_number_weighting)
  ls.build_up()
  lambdas = eigensystem.real_symmetric(
    ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
  # assert the restrained L.S. problem is not too ill-conditionned
  cond = math.log10(lambdas[0]/lambdas[-1])
  msg = "light elements: %.1f" % cond
  if verbose: print msg
  assert cond < worst_condition_number_acceptable, msg

  # one heavy element
  xs0 = random_structure.xray_structure(
    space_group_info=sgtbx.space_group_info('hall: P 2yb'),
    elements=['Zn', 'C', 'C', 'C', 'O', 'N'],
    use_u_aniso=True)
  fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
  xs = xs0.deep_copy_scatterers()
  xs.shake_adp()
  xs.shake_sites_in_place(rms_difference=0.1)
  for sc in xs.scatterers():
    sc.flags.set_grad_site(True).set_grad_u_aniso(True)
  ls = least_squares.crystallographic_ls(
    fo_sq.as_xray_observations(),
    constraints.reparametrisation(
      structure=xs,
      constraints=[],
      connectivity_table=smtbx.utils.connectivity_table(xs)),
    weighting_scheme=least_squares.mainstream_shelx_weighting(),
    origin_fixing_restraints_type=
    origin_fixing_restraints.atomic_number_weighting)
  ls.build_up()
  lambdas = eigensystem.real_symmetric(
    ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
  # assert the restrained L.S. problem is not too ill-conditionned
  cond = math.log10(lambdas[0]/lambdas[-1])
  msg = "one heavy element + light elements (synthetic data): %.1f" % cond
  if verbose: print msg
  assert cond < worst_condition_number_acceptable, msg

  # are esd's for x,y,z coordinates of the same order of magnitude?
  var_cart = covariance.orthogonalize_covariance_matrix(
    ls.covariance_matrix(),
    xs.unit_cell(),
    xs.parameter_map())
  var_site_cart = covariance.extract_covariance_matrix_for_sites(
      flex.size_t_range(len(xs.scatterers())),
      var_cart,
      xs.parameter_map())
  site_esds = var_site_cart.matrix_packed_u_diagonal()
  indicators = flex.double()
  for i in xrange(0, len(site_esds), 3):
    stats = scitbx.math.basic_statistics(site_esds[i:i+3])
    indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
  assert indicators.all_lt(1)

  # especially troublesome structure with one heavy element
  # (contributed by Jonathan Coome)
  xs0 = xray.structure(
    crystal_symmetry=crystal.symmetry(
      unit_cell=(8.4519, 8.4632, 18.7887, 90, 96.921, 90),
      space_group_symbol="hall: P 2yb"),
    scatterers=flex.xray_scatterer([
      xray.scatterer( #0
                      label="ZN1",
                      site=(-0.736683, -0.313978, -0.246902),
                      u=(0.000302, 0.000323, 0.000054,
                         0.000011, 0.000015, -0.000004)),
      xray.scatterer( #1
                      label="N3B",
                      site=(-0.721014, -0.313583, -0.134277),
                      u=(0.000268, 0.000237, 0.000055,
                         -0.000027, 0.000005, 0.000006)),
      xray.scatterer( #2
                      label="N3A",
                      site=(-0.733619, -0.290423, -0.357921),
                      u=(0.000229, 0.000313, 0.000053,
                         0.000022, 0.000018, -0.000018)),
      xray.scatterer( #3
                      label="C9B",
                      site=(-1.101537, -0.120157, -0.138063),
                      u=(0.000315, 0.000345, 0.000103,
                         0.000050, 0.000055, -0.000017)),
    xray.scatterer( #4
                    label="N5B",
                    site=(-0.962032, -0.220345, -0.222045),
                    u=(0.000274, 0.000392, 0.000060,
                       -0.000011, -0.000001, -0.000002)),
    xray.scatterer( #5
                    label="N1B",
                    site=(-0.498153, -0.402742, -0.208698),
                    u=(0.000252, 0.000306, 0.000063,
                       0.000000, 0.000007, 0.000018)),
    xray.scatterer( #6
                    label="C3B",
                    site=(-0.322492, -0.472610, -0.114594),
                    u=(0.000302, 0.000331, 0.000085,
                       0.000016, -0.000013, 0.000037)),
    xray.scatterer( #7
                    label="C4B",
                    site=(-0.591851, -0.368163, -0.094677),
                    u=(0.000262, 0.000255, 0.000073,
                       -0.000034, 0.000027, -0.000004)),
    xray.scatterer( #8
                    label="N4B",
                    site=(-0.969383, -0.204624, -0.150014),
                    u=(0.000279, 0.000259, 0.000070,
                       -0.000009, 0.000039, 0.000000)),
    xray.scatterer( #9
                    label="N2B",
                    site=(-0.470538, -0.414572, -0.135526),
                    u=(0.000277, 0.000282, 0.000065,
                       0.000003, 0.000021, -0.000006)),
    xray.scatterer( #10
                    label="C8A",
                    site=(-0.679889, -0.158646, -0.385629),
                    u=(0.000209, 0.000290, 0.000078,
                       0.000060, 0.000006, 0.000016)),
    xray.scatterer( #11
                    label="N5A",
                    site=(-0.649210, -0.075518, -0.263412),
                    u=(0.000307, 0.000335, 0.000057,
                       -0.000002, 0.000016, -0.000012)),
    xray.scatterer( #12
                    label="C6B",
                    site=(-0.708620, -0.325965, 0.011657),
                    u=(0.000503, 0.000318, 0.000053,
                       -0.000058, 0.000032, -0.000019)),
    xray.scatterer( #13
                    label="C10B",
                    site=(-1.179332, -0.083184, -0.202815),
                    u=(0.000280, 0.000424, 0.000136,
                       0.000094, 0.000006, 0.000013)),
    xray.scatterer( #14
                    label="N1A",
                    site=(-0.838363, -0.532191, -0.293213),
                    u=(0.000312, 0.000323, 0.000060,
                       0.000018, 0.000011, -0.000008)),
    xray.scatterer( #15
                    label="C3A",
                    site=(-0.915414, -0.671031, -0.393826),
                    u=(0.000319, 0.000384, 0.000078,
                       -0.000052, -0.000001, -0.000020)),
    xray.scatterer( #16
                    label="C1A",
                    site=(-0.907466, -0.665419, -0.276011),
                    u=(0.000371, 0.000315, 0.000079,
                       0.000006, 0.000036, 0.000033)),
    xray.scatterer( #17
                    label="C1B",
                    site=(-0.365085, -0.452753, -0.231927),
                    u=(0.000321, 0.000253, 0.000087,
                       -0.000024, 0.000047, -0.000034)),
    xray.scatterer( #18
                    label="C11A",
                    site=(-0.598622, 0.053343, -0.227354),
                    u=(0.000265, 0.000409, 0.000084,
                       0.000088, -0.000018, -0.000030)),
    xray.scatterer( #19
                    label="C2A",
                    site=(-0.958694, -0.755645, -0.337016),
                    u=(0.000394, 0.000350, 0.000106,
                       -0.000057, 0.000027, -0.000005)),
    xray.scatterer( #20
                    label="C4A",
                    site=(-0.784860, -0.407601, -0.402050),
                    u=(0.000238, 0.000296, 0.000064,
                       0.000002, 0.000011, -0.000016)),
    xray.scatterer( #21
                    label="C5A",
                    site=(-0.784185, -0.399716, -0.475491),
                    u=(0.000310, 0.000364, 0.000062,
                       0.000044, -0.000011, -0.000017)),
    xray.scatterer( #22
                    label="N4A",
                    site=(-0.630284, -0.043981, -0.333143),
                    u=(0.000290, 0.000275, 0.000074,
                       0.000021, 0.000027, 0.000013)),
    xray.scatterer( #23
                    label="C10A",
                    site=(-0.545465, 0.166922, -0.272829),
                    u=(0.000369, 0.000253, 0.000117,
                       0.000015, -0.000002, -0.000008)),
    xray.scatterer( #24
                    label="C9A",
                    site=(-0.567548, 0.102272, -0.339923),
                    u=(0.000346, 0.000335, 0.000103,
                       -0.000016, 0.000037, 0.000023)),
    xray.scatterer( #25
                    label="C11B",
                    site=(-1.089943, -0.146930, -0.253779),
                    u=(0.000262, 0.000422, 0.000102,
                       -0.000018, -0.000002, 0.000029)),
    xray.scatterer( #26
                    label="N2A",
                    site=(-0.843385, -0.537780, -0.366515),
                    u=(0.000273, 0.000309, 0.000055,
                       -0.000012, -0.000005, -0.000018)),
    xray.scatterer( #27
                    label="C7A",
                    site=(-0.674021, -0.136086, -0.457790),
                    u=(0.000362, 0.000378, 0.000074,
                       0.000043, 0.000034, 0.000016)),
    xray.scatterer( #28
                    label="C8B",
                    site=(-0.843625, -0.264182, -0.102023),
                    u=(0.000264, 0.000275, 0.000072,
                       -0.000025, 0.000019, -0.000005)),
    xray.scatterer( #29
                    label="C6A",
                    site=(-0.726731, -0.261702, -0.502366),
                    u=(0.000339, 0.000472, 0.000064,
                       0.000062, -0.000003, 0.000028)),
    xray.scatterer( #30
                    label="C5B",
                    site=(-0.577197, -0.376753, -0.020800),
                    u=(0.000349, 0.000353, 0.000066,
                       -0.000082, -0.000022, 0.000014)),
    xray.scatterer( #31
                    label="C2B",
                    site=(-0.252088, -0.497338, -0.175057),
                    u=(0.000251, 0.000342, 0.000119,
                       0.000020, 0.000034, -0.000018)),
    xray.scatterer( #32
                    label="C7B",
                    site=(-0.843956, -0.268811, -0.028080),
                    u=(0.000344, 0.000377, 0.000078,
                       -0.000029, 0.000059, -0.000007)),
    xray.scatterer( #33
                    label="F4B",
                    site=(-0.680814, -0.696808, -0.115056),
                    u=(0.000670, 0.000408, 0.000109,
                       -0.000099, 0.000139, -0.000031)),
    xray.scatterer( #34
                    label="F1B",
                    site=(-0.780326, -0.921249, -0.073962),
                    u=(0.000687, 0.000357, 0.000128,
                       -0.000152, -0.000011, 0.000021)),
    xray.scatterer( #35
                    label="B1B",
                    site=(-0.795220, -0.758128, -0.075955),
                    u=(0.000413, 0.000418, 0.000075,
                       0.000054, 0.000045, 0.000023)),
    xray.scatterer( #36
                    label="F2B",
                    site=(-0.945140, -0.714626, -0.105820),
                    u=(0.000584, 0.001371, 0.000108,
                       0.000420, 0.000067, 0.000134)),
    xray.scatterer( #37
                    label="F3B",
                    site=(-0.768914, -0.701660, -0.005161),
                    u=(0.000678, 0.000544, 0.000079,
                       -0.000000, 0.000090, -0.000021)),
    xray.scatterer( #38
                    label="F1A",
                    site=(-0.109283, -0.252334, -0.429288),
                    u=(0.000427, 0.001704, 0.000125,
                       0.000407, 0.000041, 0.000035)),
    xray.scatterer( #39
                    label="F4A",
                    site=(-0.341552, -0.262864, -0.502023),
                    u=(0.000640, 0.000557, 0.000081,
                       -0.000074, 0.000042, -0.000052)),
    xray.scatterer( #40
                    label="F3A",
                    site=(-0.324533, -0.142292, -0.393215),
                    u=(0.000471, 0.001203, 0.000134,
                       0.000333, -0.000057, -0.000220)),
    xray.scatterer( #41
                    label="F2A",
                    site=(-0.312838, -0.405405, -0.400231),
                    u=(0.002822, 0.000831, 0.000092,
                       -0.000648, 0.000115, 0.000027)),
    xray.scatterer( #42
                    label="B1A",
                    site=(-0.271589, -0.268874, -0.430724),
                    u=(0.000643, 0.000443, 0.000079,
                       0.000040, 0.000052, -0.000034)),
    xray.scatterer( #43
                    label="H5B",
                    site=(-0.475808, -0.413802, 0.004402),
                    u=0.005270),
    xray.scatterer( #44
                    label="H6B",
                    site=(-0.699519, -0.326233, 0.062781),
                    u=0.019940),
    xray.scatterer( #45
                    label="H3B",
                    site=(-0.283410, -0.484757, -0.063922),
                    u=0.029990),
    xray.scatterer( #46
                    label="H1B",
                    site=(-0.357103, -0.451819, -0.284911),
                    u=0.031070),
    xray.scatterer( #47
                    label="H10A",
                    site=(-0.495517, 0.268296, -0.256187),
                    u=0.027610),
    xray.scatterer( #48
                    label="H2B",
                    site=(-0.147129, -0.535141, -0.174699),
                    u=0.017930),
    xray.scatterer( #49
                    label="H7A",
                    site=(-0.643658, -0.031387, -0.475357),
                    u=0.020200),
    xray.scatterer( #50
                    label="H1A",
                    site=(-0.912757, -0.691043, -0.227554),
                    u=0.033320),
    xray.scatterer( #51
                    label="H7B",
                    site=(-0.933670, -0.241189, -0.010263),
                    u=0.021310),
    xray.scatterer( #52
                    label="H11B",
                    site=(-1.107736, -0.155470, -0.311996),
                    u=0.041500),
    xray.scatterer( #53
                    label="H9A",
                    site=(-0.539908, 0.139753, -0.382281),
                    u=0.007130),
    xray.scatterer( #54
                    label="H10B",
                    site=(-1.265944, -0.029610, -0.212398),
                    u=0.030910),
    xray.scatterer( #55
                    label="H3A",
                    site=(-0.934728, -0.691149, -0.450551),
                    u=0.038950),
    xray.scatterer( #56
                    label="H5A",
                    site=(-0.833654, -0.487479, -0.508239),
                    u=0.031150),
    xray.scatterer( #57
                    label="H6A",
                    site=(-0.742871, -0.242269, -0.558157),
                    u=0.050490),
    xray.scatterer( #58
                    label="H9B",
                    site=(-1.120150, -0.093752, -0.090706),
                    u=0.039310),
    xray.scatterer( #59
                    label="H11A",
                    site=(-0.593074, 0.054973, -0.180370),
                    u=0.055810),
    xray.scatterer( #60
                    label="H2A",
                    site=(-0.999576, -0.842158, -0.340837),
                    u=0.057030)
    ]))
  fo_sq = xs0.structure_factors(d_min=0.8).f_calc().norm()
  fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1.))
  for hydrogen_flag in (True, False):
    xs = xs0.deep_copy_scatterers()
    if not hydrogen_flag:
      xs.select_inplace(~xs.element_selection('H'))
    xs.shake_adp()
    xs.shake_sites_in_place(rms_difference=0.1)
    for sc in xs.scatterers():
      sc.flags.set_grad_site(True).set_grad_u_aniso(False)
    ls = least_squares.crystallographic_ls(
      fo_sq.as_xray_observations(),
      constraints.reparametrisation(
        structure=xs,
        constraints=[],
        connectivity_table=smtbx.utils.connectivity_table(xs)),
      weighting_scheme=least_squares.unit_weighting(),
      origin_fixing_restraints_type=
      origin_fixing_restraints.atomic_number_weighting)

    ls.build_up()
    lambdas = eigensystem.real_symmetric(
      ls.normal_matrix_packed_u().matrix_packed_u_as_symmetric()).values()
    # assert the restrained L.S. problem is not too ill-conditionned
    cond = math.log10(lambdas[0]/lambdas[-1])
    msg = ("one heavy element + light elements (real data) %s Hydrogens: %.1f"
           % (['without', 'with'][hydrogen_flag], cond))
    if verbose: print msg
    assert cond < worst_condition_number_acceptable, msg


    # are esd's for x,y,z coordinates of the same order of magnitude?
    var_cart = covariance.orthogonalize_covariance_matrix(
      ls.covariance_matrix(),
      xs.unit_cell(),
      xs.parameter_map())
    var_site_cart = covariance.extract_covariance_matrix_for_sites(
        flex.size_t_range(len(xs.scatterers())),
        var_cart,
        xs.parameter_map())
    site_esds = var_site_cart.matrix_packed_u_diagonal()
    indicators = flex.double()
    for i in xrange(0, len(site_esds), 3):
      stats = scitbx.math.basic_statistics(site_esds[i:i+3])
      indicators.append(stats.bias_corrected_standard_deviation/stats.mean)
    assert indicators.all_lt(1)