def scale_frame_detail(self, result, file_name, db_mgr, out):
        # If the pickled integration file does not contain a wavelength,
        # fall back on the value given on the command line.  XXX The
        # wavelength parameter should probably be removed from master_phil
        # once all pickled integration files contain it.
        if ("wavelength" in result):
            wavelength = result["wavelength"]
        elif (self.params.wavelength is not None):
            wavelength = self.params.wavelength
        else:
            # XXX Give error, or raise exception?
            return None
        assert (wavelength > 0)

        observations = result["observations"][0]
        cos_two_polar_angle = result["cos_two_polar_angle"]

        assert observations.size() == cos_two_polar_angle.size()
        tt_vec = observations.two_theta(wavelength)
        #print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi
        cos_tt_vec = flex.cos(tt_vec.data())
        sin_tt_vec = flex.sin(tt_vec.data())
        cos_sq_tt_vec = cos_tt_vec * cos_tt_vec
        sin_sq_tt_vec = sin_tt_vec * sin_tt_vec
        P_nought_vec = 0.5 * (1. + cos_sq_tt_vec)

        F_prime = -1.0  # Hard-coded value defines the incident polarization axis
        P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec
        # XXX added as a diagnostic
        prange = P_nought_vec - P_prime

        other_F_prime = 1.0
        otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec
        otherprange = P_nought_vec - otherP_prime
        diff2 = flex.abs(prange - otherprange)
        print "mean diff is", flex.mean(diff2), "range", flex.min(
            diff2), flex.max(diff2)
        # XXX done
        observations = observations / (P_nought_vec - P_prime)
        # This corrects observations for polarization assuming 100% polarization on
        # one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0)
        # Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre
        # (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15.

        print "Step 3. Correct for polarization."
        indexed_cell = observations.unit_cell()

        observations_original_index = observations.deep_copy()
        if result.get(
                "model_partialities", None
        ) is not None and result["model_partialities"][0] is not None:
            # some recordkeeping useful for simulations
            partialities_original_index = observations.customized_copy(
                crystal_symmetry=self.miller_set.crystal_symmetry(),
                data=result["model_partialities"][0]["data"],
                sigmas=flex.double(result["model_partialities"][0]
                                   ["data"].size()),  #dummy value for sigmas
                indices=result["model_partialities"][0]["indices"],
            ).resolution_filter(d_min=self.params.d_min)

        assert len(observations_original_index.indices()) == len(
            observations.indices())

        # Now manipulate the data to conform to unit cell, asu, and space group
        # of reference.  The resolution will be cut later.
        # Only works if there is NOT an indexing ambiguity!
        observations = observations.customized_copy(
            anomalous_flag=not self.params.merge_anomalous,
            crystal_symmetry=self.miller_set.crystal_symmetry()).map_to_asu()

        observations_original_index = observations_original_index.customized_copy(
            anomalous_flag=not self.params.merge_anomalous,
            crystal_symmetry=self.miller_set.crystal_symmetry())
        print "Step 4. Filter on global resolution and map to asu"
        print >> out, "Data in reference setting:"
        #observations.show_summary(f=out, prefix="  ")
        show_observations(observations, out=out)

        #if self.params.significance_filter.apply is True:
        #  raise Exception("significance filter not implemented in samosa")
        if self.params.significance_filter.apply is True:  #------------------------------------
            # Apply an I/sigma filter ... accept resolution bins only if they
            #   have significant signal; tends to screen out higher resolution observations
            #   if the integration model doesn't quite fit
            N_obs_pre_filter = observations.size()
            N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct
            N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct

            # Ensure there is at least one bin.
            N_bins = max([
                min([self.params.significance_filter.n_bins,
                     N_bins_small_set]), N_bins_large_set, 1
            ])
            print "Total obs %d Choose n bins = %d" % (N_obs_pre_filter,
                                                       N_bins)
            bin_results = show_observations(observations,
                                            out=out,
                                            n_bins=N_bins)
            #show_observations(observations, out=sys.stdout, n_bins=N_bins)
            acceptable_resolution_bins = [
                bin.mean_I_sigI > self.params.significance_filter.sigma
                for bin in bin_results
            ]
            acceptable_nested_bin_sequences = [
                i for i in xrange(len(acceptable_resolution_bins))
                if False not in acceptable_resolution_bins[:i + 1]
            ]
            if len(acceptable_nested_bin_sequences) == 0:
                return null_data(file_name=file_name,
                                 log_out=out.getvalue(),
                                 low_signal=True)
            else:
                N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
                imposed_res_filter = float(bin_results[N_acceptable_bins -
                                                       1].d_range.split()[2])
                imposed_res_sel = observations.resolution_filter_selection(
                    d_min=imposed_res_filter)
                observations = observations.select(imposed_res_sel)
                observations_original_index = observations_original_index.select(
                    imposed_res_sel)
                print "New resolution filter at %7.2f" % imposed_res_filter, file_name
            print "N acceptable bins", N_acceptable_bins
            print "Old n_obs: %d, new n_obs: %d" % (N_obs_pre_filter,
                                                    observations.size())
            print "Step 5. Frame by frame resolution filter"
            # Finished applying the binwise I/sigma filter---------------------------------------

        if self.params.raw_data.sdfac_auto is True:
            raise Exception("sdfac auto not implemented in samosa.")

        print "Step 6.  Match to reference intensities, filter by correlation, filter out negative intensities."
        assert len(observations_original_index.indices()) \
          ==   len(observations.indices())

        data = frame_data(self.n_refl, file_name)
        data.set_indexed_cell(indexed_cell)
        data.d_min = observations.d_min()

        # Ensure that match_multi_indices() will return identical results
        # when a frame's observations are matched against the
        # pre-generated Miller set, self.miller_set, and the reference
        # data set, self.i_model.  The implication is that the same match
        # can be used to map Miller indices to array indices for intensity
        # accumulation, and for determination of the correlation
        # coefficient in the presence of a scaling reference.
        if self.i_model is not None:
            assert len(self.i_model.indices()) == len(self.miller_set.indices()) \
              and  (self.i_model.indices() ==
                    self.miller_set.indices()).count(False) == 0

        matches = miller.match_multi_indices(
            miller_indices_unique=self.miller_set.indices(),
            miller_indices=observations.indices())

        use_weights = False  # New facility for getting variance-weighted correlation
        if self.params.scaling.algorithm in ['mark1', 'levmar']:
            # Because no correlation is computed, the correlation
            # coefficient is fixed at zero.  Setting slope = 1 means
            # intensities are added without applying a scale factor.
            sum_x = 0
            sum_y = 0
            for pair in matches.pairs():
                data.n_obs += 1
                if not self.params.include_negatives and observations.data()[
                        pair[1]] <= 0:
                    data.n_rejected += 1
                else:
                    sum_y += observations.data()[pair[1]]
            N = data.n_obs - data.n_rejected

        # Early return if there are no positive reflections on the frame.
        if data.n_obs <= data.n_rejected:
            return null_data(file_name=file_name,
                             log_out=out.getvalue(),
                             low_signal=True)

        # Update the count for each matched reflection.  This counts
        # reflections with non-positive intensities, too.
        data.completeness += matches.number_of_matches(0).as_int()
        data.wavelength = wavelength

        if not self.params.scaling.enable:  # Do not scale anything
            print "Scale factor to an isomorphous reference PDB will NOT be applied."
            slope = 1.0
            offset = 0.0

        observations_original_index_indices = observations_original_index.indices(
        )
        if db_mgr is None:
            return unpack(MINI.x)  # special exit for two-color indexing

        kwargs = {
            'wavelength': wavelength,
            'beam_x': result['xbeam'],
            'beam_y': result['ybeam'],
            'distance': result['distance'],
            'unique_file_name': data.file_name
        }

        ORI = result["current_orientation"][0]
        Astar = matrix.sqr(ORI.reciprocal_matrix())

        kwargs['res_ori_1'] = Astar[0]
        kwargs['res_ori_2'] = Astar[1]
        kwargs['res_ori_3'] = Astar[2]
        kwargs['res_ori_4'] = Astar[3]
        kwargs['res_ori_5'] = Astar[4]
        kwargs['res_ori_6'] = Astar[5]
        kwargs['res_ori_7'] = Astar[6]
        kwargs['res_ori_8'] = Astar[7]
        kwargs['res_ori_9'] = Astar[8]
        assert self.params.scaling.report_ML is True
        kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0]
        kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0]

        frame_id_0_base = db_mgr.insert_frame(**kwargs)

        xypred = result["mapped_predictions"][0]
        indices = flex.size_t([pair[1] for pair in matches.pairs()])

        sel_observations = flex.intersection(size=observations.data().size(),
                                             iselections=[indices])
        set_original_hkl = observations_original_index_indices.select(
            flex.intersection(size=observations_original_index_indices.size(),
                              iselections=[indices]))
        set_xypred = xypred.select(
            flex.intersection(size=xypred.size(), iselections=[indices]))

        kwargs = {
            'hkl_id_0_base': [pair[0] for pair in matches.pairs()],
            'i': observations.data().select(sel_observations),
            'sigi': observations.sigmas().select(sel_observations),
            'detector_x': [xy[0] for xy in set_xypred],
            'detector_y': [xy[1] for xy in set_xypred],
            'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()),
            'overload_flag': [0] * len(matches.pairs()),
            'original_h': [hkl[0] for hkl in set_original_hkl],
            'original_k': [hkl[1] for hkl in set_original_hkl],
            'original_l': [hkl[2] for hkl in set_original_hkl]
        }

        db_mgr.insert_observation(**kwargs)

        print >> out, "Lattice: %d reflections" % (data.n_obs -
                                                   data.n_rejected)
        print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \
          "average calc", sum_x / (data.n_obs - data.n_rejected)
        print >> out, "Rejected %d reflections with negative intensities" % \
            data.n_rejected

        data.accept = True
        for pair in matches.pairs():
            if not self.params.include_negatives and (
                    observations.data()[pair[1]] <= 0):
                continue
            Intensity = observations.data()[pair[1]]
            # Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed.
            if Intensity == 0:
                continue

            # Add the reflection as a two-tuple of intensity and I/sig(I)
            # to the dictionary of observations.
            index = self.miller_set.indices()[pair[0]]
            isigi = (Intensity, observations.data()[pair[1]] /
                     observations.sigmas()[pair[1]], 1.0)
            if index in data.ISIGI:
                data.ISIGI[index].append(isigi)
            else:
                data.ISIGI[index] = [isigi]

            sigma = observations.sigmas()[pair[1]]
            variance = sigma * sigma
            data.summed_N[pair[0]] += 1
            data.summed_wt_I[pair[0]] += Intensity / variance
            data.summed_weight[pair[0]] += 1 / variance

        data.set_log_out(out.getvalue())
        return data
  def scale_frame_detail(self,timestamp,cursor,do_inserts=True,result=None):#, file_name, db_mgr, out):
    if result is None:
      result = self.params

    # If the pickled integration file does not contain a wavelength,
    # fall back on the value given on the command line.  XXX The
    # wavelength parameter should probably be removed from master_phil
    # once all pickled integration files contain it.
    wavelength = result["wavelength"]
    assert (wavelength > 0)

    # Do not apply polarization correction here, as this requires knowledge of
    # pixel size at minimum, and full detector geometry in general.  The optimal
    # redesign would be to apply the polarization correction just after the integration
    # step in the integration code.
    print "Step 3. Correct for polarization."
    observations = result["observations"][0]
    indexed_cell = observations.unit_cell()

    observations_original_index = observations.deep_copy()

    assert len(observations_original_index.indices()) == len(observations.indices())

    # Now manipulate the data to conform to unit cell, asu, and space group
    # of reference.  The resolution will be cut later.
    # Only works if there is NOT an indexing ambiguity!
    #observations = observations.customized_copy(
    #  anomalous_flag=not self.params.merge_anomalous,
    #  crystal_symmetry=self.miller_set.crystal_symmetry()
    #  ).map_to_asu()

    #observations_original_index = observations_original_index.customized_copy(
    #  anomalous_flag=not self.params.merge_anomalous,
    #  crystal_symmetry=self.miller_set.crystal_symmetry()
    #  )
    observations = observations.customized_copy(anomalous_flag=False).map_to_asu()
    print "Step 4. Filter on global resolution and map to asu"

    #observations.show_summary(f=out, prefix="  ")
    from rstbx.dials_core.integration_core import show_observations
    show_observations(observations)


    print "Step 6.  Match to reference intensities, filter by correlation, filter out negative intensities."
    assert len(observations_original_index.indices()) \
      ==   len(observations.indices())

    # Ensure that match_multi_indices() will return identical results
    # when a frame's observations are matched against the
    # pre-generated Miller set, self.miller_set, and the reference
    # data set, self.i_model.  The implication is that the same match
    # can be used to map Miller indices to array indices for intensity
    # accumulation, and for determination of the correlation
    # coefficient in the presence of a scaling reference.
    self.miller_set.show_summary(prefix="mset ")

    matches = match_multi_indices(
      miller_indices_unique=self.miller_set.indices(),
      miller_indices=observations.indices())

    slope = 1.0
    offset = 0.0

    print result.get("sa_parameters")[0]
    have_sa_params = ( type(result.get("sa_parameters")[0]) == type(dict()) )

    observations_original_index_indices = observations_original_index.indices()
    print result.keys()
    kwargs = {'wavelength': wavelength,
              'beam_x': result['xbeam'],
              'beam_y': result['ybeam'],
              'distance': result['distance'],
              'slope': slope,
              'offset': offset,
              'unique_file_name': timestamp,
              'eventstamp':timestamp,
              'sifoil': 0.0}

    trial_id = self.get_trial_id(cursor)
    run_id = self.get_run_id(cursor)
    kwargs["trials_id"] = trial_id
    kwargs["rungroups_id"] = self.rungroup_id
    kwargs["runs_run_id"] = run_id
    kwargs["isoforms_isoform_id"] = self.isoform_id
    res_ori_direct = matrix.sqr(
        observations.unit_cell().orthogonalization_matrix()).transpose().elems

    kwargs['res_ori_1'] = res_ori_direct[0]
    kwargs['res_ori_2'] = res_ori_direct[1]
    kwargs['res_ori_3'] = res_ori_direct[2]
    kwargs['res_ori_4'] = res_ori_direct[3]
    kwargs['res_ori_5'] = res_ori_direct[4]
    kwargs['res_ori_6'] = res_ori_direct[5]
    kwargs['res_ori_7'] = res_ori_direct[6]
    kwargs['res_ori_8'] = res_ori_direct[7]
    kwargs['res_ori_9'] = res_ori_direct[8]

    kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",[float("NaN")])[0]
    kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",[float("NaN")])[0]
    kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",[float("NaN")])[0]


    sql, parameters = self._insert(
      table='`%s_frames`' % self.db_experiment_tag,
      **kwargs)
    print sql
    print parameters
    results = {'frame':[sql, parameters, kwargs]}
    if do_inserts:
      cursor.execute(sql, parameters[0])
      frame_id = cursor.lastrowid
    else:
      frame_id = None

    xypred = result["mapped_predictions"][0]
    indices = flex.size_t([pair[1] for pair in matches.pairs()])

    sel_observations = flex.intersection(
      size=observations.data().size(),
      iselections=[indices])
    set_original_hkl = observations_original_index_indices.select(
      flex.intersection(
        size=observations_original_index_indices.size(),
        iselections=[indices]))
    set_xypred = xypred.select(
      flex.intersection(
        size=xypred.size(),
        iselections=[indices]))
    ''' debugging printout
    print len(observations.data())
    print len(indices)
    print len(sel_observations)
    for x in xrange(len(observations.data())):
      print x,observations.indices().select(sel_observations)[x],
      print set_original_hkl[x],
      index_into_hkl_id = matches.pairs()[x][0]
      print index_into_hkl_id,
      print self.miller_set.indices()[index_into_hkl_id],
      cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%(
            self.db_experiment_tag, self.miller_set_id[index_into_hkl_id]))

      print cursor.fetchall()[0]
    '''
    print "Adding %d observations for this frame"%(len(sel_observations))
    kwargs = {'hkls_id': self.miller_set_id.select(flex.size_t([pair[0] for pair in matches.pairs()])),
              'i': observations.data().select(sel_observations),
              'sigi': observations.sigmas().select(sel_observations),
              'detector_x_px': [xy[0] for xy in set_xypred],
              'detector_y_px': [xy[1] for xy in set_xypred],
              'frames_id': [frame_id] * len(matches.pairs()),
              'overload_flag': [0] * len(matches.pairs()),
              'original_h': [hkl[0] for hkl in set_original_hkl],
              'original_k': [hkl[1] for hkl in set_original_hkl],
              'original_l': [hkl[2] for hkl in set_original_hkl],
              'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()),
              'frames_trials_id': [trial_id] * len(matches.pairs()),
              'panel': [0] * len(matches.pairs())
    }
    if do_inserts:
      # For MySQLdb executemany() is six times slower than a single big
      # execute() unless the "values" keyword is given in lowercase
      # (http://sourceforge.net/p/mysql-python/bugs/305).
      #
      # See also merging_database_sqlite3._insert()
      query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \
              + ", ".join(kwargs.keys()) + ") values (" \
              + ", ".join(["%s"] * len(kwargs.keys())) + ")"
      try:
        parameters = zip(*kwargs.values())
      except TypeError:
        parameters = [kwargs.values()]
      cursor.executemany(query, parameters)
      #print "done execute many"
      #print cursor._last_executed
      results['observations'] = [query, parameters, kwargs]
    else:
      # since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results
      results['observations'] = [None, None, kwargs]

    return results
Beispiel #3
0
    def scale_frame_detail(self,
                           timestamp,
                           cursor,
                           do_inserts=True,
                           result=None):  #, file_name, db_mgr, out):
        if result is None:
            result = self.params

        # If the pickled integration file does not contain a wavelength,
        # fall back on the value given on the command line.  XXX The
        # wavelength parameter should probably be removed from master_phil
        # once all pickled integration files contain it.
        wavelength = result["wavelength"]
        assert (wavelength > 0)

        # Do not apply polarization correction here, as this requires knowledge of
        # pixel size at minimum, and full detector geometry in general.  The optimal
        # redesign would be to apply the polarization correction just after the integration
        # step in the integration code.
        print("Step 3. Correct for polarization.")
        observations = result["observations"][0]
        indexed_cell = observations.unit_cell()

        observations_original_index = observations.deep_copy()

        assert len(observations_original_index.indices()) == len(
            observations.indices())

        # Now manipulate the data to conform to unit cell, asu, and space group
        # of reference.  The resolution will be cut later.
        # Only works if there is NOT an indexing ambiguity!
        #observations = observations.customized_copy(
        #  anomalous_flag=not self.params.merge_anomalous,
        #  crystal_symmetry=self.miller_set.crystal_symmetry()
        #  ).map_to_asu()

        #observations_original_index = observations_original_index.customized_copy(
        #  anomalous_flag=not self.params.merge_anomalous,
        #  crystal_symmetry=self.miller_set.crystal_symmetry()
        #  )
        observations = observations.customized_copy(
            anomalous_flag=False).map_to_asu()
        print("Step 4. Filter on global resolution and map to asu")

        #observations.show_summary(f=out, prefix="  ")
        from rstbx.dials_core.integration_core import show_observations
        show_observations(observations)

        print(
            "Step 6.  Match to reference intensities, filter by correlation, filter out negative intensities."
        )
        assert len(observations_original_index.indices()) \
          ==   len(observations.indices())

        # Ensure that match_multi_indices() will return identical results
        # when a frame's observations are matched against the
        # pre-generated Miller set, self.miller_set, and the reference
        # data set, self.i_model.  The implication is that the same match
        # can be used to map Miller indices to array indices for intensity
        # accumulation, and for determination of the correlation
        # coefficient in the presence of a scaling reference.
        self.miller_set.show_summary(prefix="mset ")

        matches = match_multi_indices(
            miller_indices_unique=self.miller_set.indices(),
            miller_indices=observations.indices())

        slope = 1.0
        offset = 0.0

        print(result.get("sa_parameters")[0])
        have_sa_params = (type(result.get("sa_parameters")[0]) == type(dict()))

        observations_original_index_indices = observations_original_index.indices(
        )
        print(list(result.keys()))
        kwargs = {
            'wavelength': wavelength,
            'beam_x': result['xbeam'],
            'beam_y': result['ybeam'],
            'distance': result['distance'],
            'slope': slope,
            'offset': offset,
            'unique_file_name': timestamp,
            'eventstamp': timestamp,
            'sifoil': 0.0
        }

        trial_id = self.get_trial_id(cursor)
        run_id = self.get_run_id(cursor)
        kwargs["trials_id"] = trial_id
        kwargs["rungroups_id"] = self.rungroup_id
        kwargs["runs_run_id"] = run_id
        kwargs["isoforms_isoform_id"] = self.isoform_id
        res_ori_direct = matrix.sqr(observations.unit_cell(
        ).orthogonalization_matrix()).transpose().elems

        kwargs['res_ori_1'] = res_ori_direct[0]
        kwargs['res_ori_2'] = res_ori_direct[1]
        kwargs['res_ori_3'] = res_ori_direct[2]
        kwargs['res_ori_4'] = res_ori_direct[3]
        kwargs['res_ori_5'] = res_ori_direct[4]
        kwargs['res_ori_6'] = res_ori_direct[5]
        kwargs['res_ori_7'] = res_ori_direct[6]
        kwargs['res_ori_8'] = res_ori_direct[7]
        kwargs['res_ori_9'] = res_ori_direct[8]

        kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",
                                                     [float("NaN")])[0]
        kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",
                                                 [float("NaN")])[0]
        kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",
                                                     [float("NaN")])[0]

        sql, parameters = self._insert(table='`%s_frames`' %
                                       self.db_experiment_tag,
                                       **kwargs)
        print(sql)
        print(parameters)
        results = {'frame': [sql, parameters, kwargs]}
        if do_inserts:
            cursor.execute(sql, parameters[0])
            frame_id = cursor.lastrowid
        else:
            frame_id = None

        xypred = result["mapped_predictions"][0]
        indices = flex.size_t([pair[1] for pair in matches.pairs()])

        sel_observations = flex.intersection(size=observations.data().size(),
                                             iselections=[indices])
        set_original_hkl = observations_original_index_indices.select(
            flex.intersection(size=observations_original_index_indices.size(),
                              iselections=[indices]))
        set_xypred = xypred.select(
            flex.intersection(size=xypred.size(), iselections=[indices]))
        ''' debugging printout
    print len(observations.data())
    print len(indices)
    print len(sel_observations)
    for x in range(len(observations.data())):
      print x,observations.indices().select(sel_observations)[x],
      print set_original_hkl[x],
      index_into_hkl_id = matches.pairs()[x][0]
      print index_into_hkl_id,
      print self.miller_set.indices()[index_into_hkl_id],
      cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%(
            self.db_experiment_tag, self.miller_set_id[index_into_hkl_id]))

      print cursor.fetchall()[0]
    '''
        print("Adding %d observations for this frame" %
              (len(sel_observations)))
        kwargs = {
            'hkls_id':
            self.miller_set_id.select(
                flex.size_t([pair[0] for pair in matches.pairs()])),
            'i':
            observations.data().select(sel_observations),
            'sigi':
            observations.sigmas().select(sel_observations),
            'detector_x_px': [xy[0] for xy in set_xypred],
            'detector_y_px': [xy[1] for xy in set_xypred],
            'frames_id': [frame_id] * len(matches.pairs()),
            'overload_flag': [0] * len(matches.pairs()),
            'original_h': [hkl[0] for hkl in set_original_hkl],
            'original_k': [hkl[1] for hkl in set_original_hkl],
            'original_l': [hkl[2] for hkl in set_original_hkl],
            'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()),
            'frames_trials_id': [trial_id] * len(matches.pairs()),
            'panel': [0] * len(matches.pairs())
        }
        if do_inserts:
            # For MySQLdb executemany() is six times slower than a single big
            # execute() unless the "values" keyword is given in lowercase
            # (http://sourceforge.net/p/mysql-python/bugs/305).
            #
            # See also merging_database_sqlite3._insert()
            query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \
                    + ", ".join(kwargs) + ") values (" \
                    + ", ".join(["%s"] * len(kwargs)) + ")"
            try:
                parameters = list(zip(*list(kwargs.values())))
            except TypeError:
                parameters = [list(kwargs.values())]
            cursor.executemany(query, parameters)
            #print "done execute many"
            #print cursor._last_executed
            results['observations'] = [query, parameters, kwargs]
        else:
            # since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results
            results['observations'] = [None, None, kwargs]

        return results
Beispiel #4
0
  def scale_frame_detail(self, result, file_name, db_mgr, out):
    # If the pickled integration file does not contain a wavelength,
    # fall back on the value given on the command line.  XXX The
    # wavelength parameter should probably be removed from master_phil
    # once all pickled integration files contain it.
    if (result.has_key("wavelength")):
      wavelength = result["wavelength"]
    elif (self.params.wavelength is not None):
      wavelength = self.params.wavelength
    else:
      # XXX Give error, or raise exception?
      return None
    assert (wavelength > 0)

    observations = result["observations"][0]
    cos_two_polar_angle = result["cos_two_polar_angle"]

    assert observations.size() == cos_two_polar_angle.size()
    tt_vec = observations.two_theta(wavelength)
    #print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi
    cos_tt_vec = flex.cos( tt_vec.data() )
    sin_tt_vec = flex.sin( tt_vec.data() )
    cos_sq_tt_vec = cos_tt_vec * cos_tt_vec
    sin_sq_tt_vec = sin_tt_vec * sin_tt_vec
    P_nought_vec = 0.5 * (1. + cos_sq_tt_vec)

    F_prime = -1.0 # Hard-coded value defines the incident polarization axis
    P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec
    # XXX added as a diagnostic
    prange=P_nought_vec - P_prime

    other_F_prime = 1.0
    otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec
    otherprange=P_nought_vec - otherP_prime
    diff2 = flex.abs(prange - otherprange)
    print "mean diff is",flex.mean(diff2), "range",flex.min(diff2), flex.max(diff2)
    # XXX done
    observations = observations / ( P_nought_vec - P_prime )
    # This corrects observations for polarization assuming 100% polarization on
    # one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0)
    # Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre
    # (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15.

    print "Step 3. Correct for polarization."
    indexed_cell = observations.unit_cell()

    observations_original_index = observations.deep_copy()
    if result.get("model_partialities",None) is not None and result["model_partialities"][0] is not None:
      # some recordkeeping useful for simulations
      partialities_original_index = observations.customized_copy(
        crystal_symmetry=self.miller_set.crystal_symmetry(),
        data = result["model_partialities"][0]["data"],
        sigmas = flex.double(result["model_partialities"][0]["data"].size()), #dummy value for sigmas
        indices = result["model_partialities"][0]["indices"],
        ).resolution_filter(d_min=self.params.d_min)

    assert len(observations_original_index.indices()) == len(observations.indices())

    # Now manipulate the data to conform to unit cell, asu, and space group
    # of reference.  The resolution will be cut later.
    # Only works if there is NOT an indexing ambiguity!
    observations = observations.customized_copy(
      anomalous_flag=not self.params.merge_anomalous,
      crystal_symmetry=self.miller_set.crystal_symmetry()
      ).map_to_asu()

    observations_original_index = observations_original_index.customized_copy(
      anomalous_flag=not self.params.merge_anomalous,
      crystal_symmetry=self.miller_set.crystal_symmetry()
      )
    print "Step 4. Filter on global resolution and map to asu"
    print >> out, "Data in reference setting:"
    #observations.show_summary(f=out, prefix="  ")
    show_observations(observations, out=out)

    #if self.params.significance_filter.apply is True:
    #  raise Exception("significance filter not implemented in samosa")
    if self.params.significance_filter.apply is True: #------------------------------------
      # Apply an I/sigma filter ... accept resolution bins only if they
      #   have significant signal; tends to screen out higher resolution observations
      #   if the integration model doesn't quite fit
      N_obs_pre_filter = observations.size()
      N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct
      N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct

      # Ensure there is at least one bin.
      N_bins = max(
        [min([self.params.significance_filter.n_bins,N_bins_small_set]),
         N_bins_large_set, 1]
      )
      print "Total obs %d Choose n bins = %d"%(N_obs_pre_filter,N_bins)
      bin_results = show_observations(observations, out=out, n_bins=N_bins)
      #show_observations(observations, out=sys.stdout, n_bins=N_bins)
      acceptable_resolution_bins = [
        bin.mean_I_sigI > self.params.significance_filter.sigma for bin in bin_results]
      acceptable_nested_bin_sequences = [i for i in xrange(len(acceptable_resolution_bins))
                                         if False not in acceptable_resolution_bins[:i+1]]
      if len(acceptable_nested_bin_sequences)==0:
        return null_data(
          file_name=file_name, log_out=out.getvalue(), low_signal=True)
      else:
        N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
        imposed_res_filter = float(bin_results[N_acceptable_bins-1].d_range.split()[2])
        imposed_res_sel = observations.resolution_filter_selection(
          d_min=imposed_res_filter)
        observations = observations.select(
          imposed_res_sel)
        observations_original_index = observations_original_index.select(
          imposed_res_sel)
        print "New resolution filter at %7.2f"%imposed_res_filter,file_name
      print "N acceptable bins",N_acceptable_bins
      print "Old n_obs: %d, new n_obs: %d"%(N_obs_pre_filter,observations.size())
      print "Step 5. Frame by frame resolution filter"
      # Finished applying the binwise I/sigma filter---------------------------------------

    if self.params.raw_data.sdfac_auto is True:
      raise Exception("sdfac auto not implemented in samosa.")

    print "Step 6.  Match to reference intensities, filter by correlation, filter out negative intensities."
    assert len(observations_original_index.indices()) \
      ==   len(observations.indices())

    data = frame_data(self.n_refl, file_name)
    data.set_indexed_cell(indexed_cell)
    data.d_min = observations.d_min()

    # Ensure that match_multi_indices() will return identical results
    # when a frame's observations are matched against the
    # pre-generated Miller set, self.miller_set, and the reference
    # data set, self.i_model.  The implication is that the same match
    # can be used to map Miller indices to array indices for intensity
    # accumulation, and for determination of the correlation
    # coefficient in the presence of a scaling reference.
    if self.i_model is not None:
      assert len(self.i_model.indices()) == len(self.miller_set.indices()) \
        and  (self.i_model.indices() ==
              self.miller_set.indices()).count(False) == 0

    matches = miller.match_multi_indices(
      miller_indices_unique=self.miller_set.indices(),
      miller_indices=observations.indices())

    use_weights = False # New facility for getting variance-weighted correlation
    if self.params.scaling.algorithm in ['mark1','levmar']:
      # Because no correlation is computed, the correlation
      # coefficient is fixed at zero.  Setting slope = 1 means
      # intensities are added without applying a scale factor.
      sum_x = 0
      sum_y = 0
      for pair in matches.pairs():
        data.n_obs += 1
        if not self.params.include_negatives and observations.data()[pair[1]] <= 0:
          data.n_rejected += 1
        else:
          sum_y += observations.data()[pair[1]]
      N = data.n_obs - data.n_rejected

    # Early return if there are no positive reflections on the frame.
    if data.n_obs <= data.n_rejected:
      return null_data(
        file_name=file_name, log_out=out.getvalue(), low_signal=True)

    # Update the count for each matched reflection.  This counts
    # reflections with non-positive intensities, too.
    data.completeness += matches.number_of_matches(0).as_int()
    data.wavelength = wavelength

    if not self.params.scaling.enable: # Do not scale anything
      print "Scale factor to an isomorphous reference PDB will NOT be applied."
      slope = 1.0
      offset = 0.0

    observations_original_index_indices = observations_original_index.indices()
    if db_mgr is None: return unpack(MINI.x) # special exit for two-color indexing

    kwargs = {'wavelength': wavelength,
              'beam_x': result['xbeam'],
              'beam_y': result['ybeam'],
              'distance': result['distance'],
              'unique_file_name': data.file_name}

    ORI = result["current_orientation"][0]
    Astar = matrix.sqr(ORI.reciprocal_matrix())

    kwargs['res_ori_1'] = Astar[0]
    kwargs['res_ori_2'] = Astar[1]
    kwargs['res_ori_3'] = Astar[2]
    kwargs['res_ori_4'] = Astar[3]
    kwargs['res_ori_5'] = Astar[4]
    kwargs['res_ori_6'] = Astar[5]
    kwargs['res_ori_7'] = Astar[6]
    kwargs['res_ori_8'] = Astar[7]
    kwargs['res_ori_9'] = Astar[8]
    assert self.params.scaling.report_ML is True
    kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0]
    kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0]

    frame_id_0_base = db_mgr.insert_frame(**kwargs)

    xypred = result["mapped_predictions"][0]
    indices = flex.size_t([pair[1] for pair in matches.pairs()])

    sel_observations = flex.intersection(
      size=observations.data().size(),
      iselections=[indices])
    set_original_hkl = observations_original_index_indices.select(
      flex.intersection(
        size=observations_original_index_indices.size(),
        iselections=[indices]))
    set_xypred = xypred.select(
      flex.intersection(
        size=xypred.size(),
        iselections=[indices]))

    kwargs = {'hkl_id_0_base': [pair[0] for pair in matches.pairs()],
              'i': observations.data().select(sel_observations),
              'sigi': observations.sigmas().select(sel_observations),
              'detector_x': [xy[0] for xy in set_xypred],
              'detector_y': [xy[1] for xy in set_xypred],
              'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()),
              'overload_flag': [0] * len(matches.pairs()),
              'original_h': [hkl[0] for hkl in set_original_hkl],
              'original_k': [hkl[1] for hkl in set_original_hkl],
              'original_l': [hkl[2] for hkl in set_original_hkl]}

    db_mgr.insert_observation(**kwargs)

    print >> out, "Lattice: %d reflections" % (data.n_obs - data.n_rejected)
    print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \
      "average calc", sum_x / (data.n_obs - data.n_rejected)
    print >> out, "Rejected %d reflections with negative intensities" % \
        data.n_rejected

    data.accept = True
    for pair in matches.pairs():
      if not self.params.include_negatives and (observations.data()[pair[1]] <= 0) :
        continue
      Intensity = observations.data()[pair[1]]
      # Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed.
      if Intensity == 0:
        continue

      # Add the reflection as a two-tuple of intensity and I/sig(I)
      # to the dictionary of observations.
      index = self.miller_set.indices()[pair[0]]
      isigi = (Intensity,
               observations.data()[pair[1]] / observations.sigmas()[pair[1]],
               1.0)
      if index in data.ISIGI:
        data.ISIGI[index].append(isigi)
      else:
        data.ISIGI[index] = [isigi]

      sigma = observations.sigmas()[pair[1]]
      variance = sigma * sigma
      data.summed_N[pair[0]] += 1
      data.summed_wt_I[pair[0]] += Intensity / variance
      data.summed_weight[pair[0]] += 1 / variance


    data.set_log_out(out.getvalue())
    return data