Пример #1
0
    def extract_shoeboxes(self,
                          imageset,
                          mask=None,
                          nthreads=1,
                          verbose=False):
        '''
    Helper function to read a load of shoebox data.

    :param imageset: The imageset
    :param mask: The mask to apply
    :param nthreads: The number of threads to use
    :param verbose: The verbosity
    :return: A tuple containing read time and extract time

    '''
        from dials.model.data import make_image
        from time import time
        assert ("shoebox" in self)
        detector = imageset.get_detector()
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))
        extractor = ShoeboxExtractor(self, len(detector), frame0, frame1)
        logger.info(" Beginning to read images")
        read_time = 0
        extract_time = 0
        for i in range(len(imageset)):
            if verbose:
                logger.info('  reading image %d' % i)
            st = time()
            image = imageset.get_corrected_data(i)
            mask2 = imageset.get_mask(i)
            if mask is not None:
                assert (len(mask) == len(mask2))
                mask2 = tuple(m1 & m2 for m1, m2 in zip(mask, mask2))
            read_time += time() - st
            st = time()
            extractor.next(make_image(image, mask2))
            extract_time += time() - st
            del image
        assert (extractor.finished())
        logger.info('  successfully read %d images' % (frame1 - frame0))
        logger.info('  read time: %g seconds' % read_time)
        logger.info('  extract time: %g seconds' % extract_time)
        return read_time, extract_time
Пример #2
0
  def extract_shoeboxes(self, imageset, mask=None, nthreads=1, verbose=False):
    '''
    Helper function to read a load of shoebox data.

    :param imageset: The imageset
    :param mask: The mask to apply
    :param nthreads: The number of threads to use
    :param verbose: The verbosity
    :return: A tuple containing read time and extract time

    '''
    from dials.model.data import make_image
    from logging import info
    from time import time
    assert("shoebox" in self)
    detector = imageset.get_detector()
    try:
      frame0, frame1 = imageset.get_array_range()
    except Exception:
      frame0, frame1 = (0, len(imageset))
    extractor = ShoeboxExtractor(self, len(detector), frame0, frame1)
    info(" Beginning to read images")
    read_time = 0
    extract_time = 0
    for i in range(len(imageset)):
      if verbose:
        info('  reading image %d' % i)
      st = time()
      image = imageset.get_corrected_data(i)
      mask2 = imageset.get_mask(i)
      if mask is not None:
        assert(len(mask) == len(mask2))
        mask2 = tuple(m1 & m2 for m1, m2 in zip(mask, mask2))
      read_time += time() - st
      st = time()
      extractor.next(make_image(image, mask2))
      extract_time += time() - st
      del image
    assert(extractor.finished())
    info('  successfully read %d images' % (frame1 - frame0))
    info('  read time: %g seconds' % read_time)
    info('  extract time: %g seconds' % extract_time)
    return read_time, extract_time
Пример #3
0
    def extract_shoeboxes(self, imageset, mask=None, nthreads=1):
        """
        Helper function to read a load of shoebox data.

        :param imageset: The imageset
        :param mask: The mask to apply
        :param nthreads: The number of threads to use
        :return: A tuple containing read time and extract time
        """
        from time import time

        from dials.model.data import make_image

        assert "shoebox" in self
        detector = imageset.get_detector()
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))
        extractor = dials_array_family_flex_ext.ShoeboxExtractor(
            self, len(detector), frame0, frame1
        )
        logger.info(" Beginning to read images")
        read_time = 0
        extract_time = 0
        for i in range(len(imageset)):
            logger.debug("  reading image %d", i)
            st = time()
            image = imageset.get_corrected_data(i)
            mask2 = imageset.get_mask(i)
            if mask is not None:
                assert len(mask) == len(mask2)
                mask2 = tuple(m1 & m2 for m1, m2 in zip(mask, mask2))
            read_time += time() - st
            st = time()
            extractor.next(make_image(image, mask2))
            extract_time += time() - st
            del image
        assert extractor.finished()
        logger.info("  successfully read %d images", frame1 - frame0)
        logger.info("  read time: %.1f seconds", read_time)
        logger.info("  extract time: %.1f seconds", extract_time)
        return read_time, extract_time
Пример #4
0
    def __call__(self):
        """
        Do the processing.

        :return: The processed data
        """
        # Set the job index
        job.index = self.index

        # Get the start time
        start_time = time()

        # Check all reflections have same imageset and get it
        exp_id = list(set(self.reflections["id"]))
        imageset = self.experiments[exp_id[0]].imageset
        for i in exp_id[1:]:
            assert (self.experiments[i].imageset == imageset
                    ), "Task can only handle 1 imageset"

        # Get the sub imageset
        frame00, frame01 = self.frames
        try:
            frame10, frame11 = imageset.get_array_range()
        except Exception:
            frame10, frame11 = (0, len(imageset))
        try:
            assert frame00 < frame01
            assert frame10 < frame11
            assert frame00 >= frame10
            assert frame01 <= frame11
            index0 = frame00 - frame10
            index1 = index0 + (frame01 - frame00)
            assert index0 < index1
            assert index0 >= 0
            assert index1 <= len(imageset)
            imageset = imageset[index0:index1]
        except Exception:
            raise RuntimeError("Programmer Error: bad array range")
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))

        # Initialise the dataset
        image_volume = MultiPanelImageVolume()
        for panel in self.experiments[0].detector:
            image_volume.add(
                ImageVolume(frame0, frame1,
                            panel.get_image_size()[1],
                            panel.get_image_size()[0]))

        # Read all the images into a block of data
        read_time = 0.0
        for i in range(len(imageset)):
            st = time()
            image = imageset.get_corrected_data(i)
            mask = imageset.get_mask(i)
            if self.params.integration.lookup.mask is not None:
                assert len(mask) == len(
                    self.params.lookup.mask
                ), "Mask/Image are incorrect size %d %d" % (
                    len(mask),
                    len(self.params.integration.lookup.mask),
                )
                mask = tuple(m1 & m2 for m1, m2 in zip(
                    self.params.integration.lookup.mask, mask))
            image_volume.set_image(frame0 + i, make_image(image, mask))
            read_time += time() - st
            del image
            del mask

        # Process the data
        st = time()
        data = self.executor.process(image_volume, self.experiments,
                                     self.reflections)
        process_time = time() - st

        # Set the result values
        return dials.algorithms.integration.Result(
            index=self.index,
            reflections=self.reflections,
            read_time=read_time,
            process_time=process_time,
            total_time=time() - start_time,
            extract_time=0,
            data=data,
        )
Пример #5
0
    def __call__(self):
        """
        Do the processing.

        :return: The processed data
        """
        # Get the start time
        start_time = time()

        # Set the global process ID
        job.index = self.index

        # Check all reflections have same imageset and get it
        exp_id = list(set(self.reflections["id"]))
        imageset = self.experiments[exp_id[0]].imageset
        for i in exp_id[1:]:
            assert (self.experiments[i].imageset == imageset
                    ), "Task can only handle 1 imageset"

        # Get the sub imageset
        frame0, frame1 = self.job

        try:
            allowed_range = imageset.get_array_range()
        except Exception:
            allowed_range = 0, len(imageset)

        try:
            # range increasing
            assert frame0 < frame1

            # within an increasing range
            assert allowed_range[1] > allowed_range[0]

            # we are processing data which is within range
            assert frame0 >= allowed_range[0]
            assert frame1 <= allowed_range[1]

            # I am 99% sure this is implied by all the code above
            assert (frame1 - frame0) <= len(imageset)
            if len(imageset) > 1:
                imageset = imageset[frame0:frame1]
        except Exception as e:
            raise RuntimeError(f"Programmer Error: bad array range: {e}")

        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))

        self.executor.initialize(frame0, frame1, self.reflections)

        # Set the shoeboxes (don't allocate)
        self.reflections["shoebox"] = flex.shoebox(
            self.reflections["panel"],
            self.reflections["bbox"],
            allocate=False,
            flatten=self.params.shoebox.flatten,
        )

        # Create the processor
        processor = ShoeboxProcessor(
            self.reflections,
            len(imageset.get_detector()),
            frame0,
            frame1,
            self.params.debug.output,
        )

        # Loop through the imageset, extract pixels and process reflections
        read_time = 0.0
        for i in range(len(imageset)):
            st = time()
            image = imageset.get_corrected_data(i)
            if imageset.is_marked_for_rejection(i):
                mask = tuple(flex.bool(im.accessor(), False) for im in image)
            else:
                mask = imageset.get_mask(i)
                if self.params.lookup.mask is not None:
                    assert len(mask) == len(
                        self.params.lookup.mask
                    ), "Mask/Image are incorrect size %d %d" % (
                        len(mask),
                        len(self.params.lookup.mask),
                    )
                    mask = tuple(
                        m1 & m2
                        for m1, m2 in zip(self.params.lookup.mask, mask))

            read_time += time() - st
            processor.next(make_image(image, mask), self.executor)
            del image
            del mask
        assert processor.finished(), "Data processor is not finished"

        # Optionally save the shoeboxes
        if self.params.debug.output and self.params.debug.separate_files:
            output = self.reflections
            if self.params.debug.select is not None:
                output = output.select(self.params.debug.select(output))
            if self.params.debug.split_experiments:
                output = output.split_by_experiment_id()
                for table in output:
                    i = table["id"][0]
                    table.as_file("shoeboxes_%d_%d.refl" % (self.index, i))
            else:
                output.as_file("shoeboxes_%d.refl" % self.index)

        # Delete the shoeboxes
        if self.params.debug.separate_files or not self.params.debug.output:
            del self.reflections["shoebox"]

        # Finalize the executor
        self.executor.finalize()

        # Return the result
        return dials.algorithms.integration.Result(
            index=self.index,
            reflections=self.reflections,
            data=self.executor.data(),
            read_time=read_time,
            extract_time=processor.extract_time(),
            process_time=processor.process_time(),
            total_time=time() - start_time,
        )
Пример #6
0
    def __call__(self):
        '''
    Do the processing.

    :return: The processed data

    '''
        from dials.array_family import flex
        from time import time
        from dials.model.data import make_image
        from libtbx.introspection import machine_memory_info

        # Get the start time
        start_time = time()

        # Set the global process ID
        job.index = self.index

        # Check all reflections have same imageset and get it
        exp_id = list(set(self.reflections['id']))
        imageset = self.experiments[exp_id[0]].imageset
        for i in exp_id[1:]:
            assert self.experiments[
                i].imageset == imageset, "Task can only handle 1 imageset"

        # Get the sub imageset
        frame00, frame01 = self.job
        try:
            frame10, frame11 = imageset.get_array_range()
        except Exception:
            frame10, frame11 = (0, len(imageset))
        try:
            assert frame00 < frame01
            assert frame10 < frame11
            assert frame00 >= frame10
            assert frame01 <= frame11
            index0 = frame00 - frame10
            index1 = index0 + (frame01 - frame00)
            assert index0 < index1
            assert index0 >= 0
            assert index1 <= len(imageset)
            imageset = imageset[index0:index1]
        except Exception:
            raise RuntimeError('Programmer Error: bad array range')
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))

        # Initlize the executor
        self.executor.initialize(frame0, frame1, self.reflections)

        # Set the shoeboxes (dont't allocate)
        self.reflections['shoebox'] = flex.shoebox(
            self.reflections['panel'],
            self.reflections['bbox'],
            allocate=False,
            flatten=self.params.shoebox.flatten)

        # Create the processor
        processor = ShoeboxProcessor(self.reflections,
                                     len(imageset.get_detector()), frame0,
                                     frame1, self.params.debug.output)

        # Compute percentage of max available. The function is not portable to
        # windows so need to add a check if the function fails. On windows no
        # warning will be printed
        memory_info = machine_memory_info()
        total_memory = memory_info.memory_total()
        sbox_memory = processor.compute_max_memory_usage()
        if total_memory is not None:
            assert total_memory > 0, "Your system appears to have no memory!"
            assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
            assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
            limit_memory = total_memory * self.params.block.max_memory_usage
            if sbox_memory > limit_memory:
                raise RuntimeError('''
        There was a problem allocating memory for shoeboxes. Possible solutions
        include increasing the percentage of memory allowed for shoeboxes or
        decreasing the block size. This could also be caused by a highly mosaic
        crystal model - is your crystal really this mosaic?
          Total system memory: %g GB
          Limit shoebox memory: %g GB
          Required shoebox memory: %g GB
        ''' % (total_memory / 1e9, limit_memory / 1e9, sbox_memory / 1e9))
            else:
                logger.info(' Memory usage:')
                logger.info('  Total system memory: %g GB' %
                            (total_memory / 1e9))
                logger.info('  Limit shoebox memory: %g GB' %
                            (limit_memory / 1e9))
                logger.info('  Required shoebox memory: %g GB' %
                            (sbox_memory / 1e9))
                logger.info('')

        # Loop through the imageset, extract pixels and process reflections
        read_time = 0.0
        for i in range(len(imageset)):
            st = time()
            image = imageset.get_corrected_data(i)
            mask = imageset.get_mask(i)
            if self.params.lookup.mask is not None:
                assert len(mask) == len(self.params.lookup.mask), \
                  "Mask/Image are incorrect size %d %d" % (
                    len(mask),
                    len(self.params.lookup.mask))
                mask = tuple(m1 & m2
                             for m1, m2 in zip(self.params.lookup.mask, mask))

            read_time += time() - st
            processor.next(make_image(image, mask), self.executor)
            del image
            del mask
        assert processor.finished(), "Data processor is not finished"

        # Optionally save the shoeboxes
        if self.params.debug.output and self.params.debug.separate_files:
            output = self.reflections
            if self.params.debug.select is not None:
                output = output.select(self.params.debug.select(output))
            if self.params.debug.split_experiments:
                output = output.split_by_experiment_id()
                for table in output:
                    i = table['id'][0]
                    table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
            else:
                output.as_pickle('shoeboxes_%d.pickle' % self.index)

        # Delete the shoeboxes
        if self.params.debug.separate_files or not self.params.debug.output:
            del self.reflections['shoebox']

        # Finalize the executor
        self.executor.finalize()

        # Return the result
        result = Result(self.index, self.reflections, self.executor.data())
        result.read_time = read_time
        result.extract_time = processor.extract_time()
        result.process_time = processor.process_time()
        result.total_time = time() - start_time
        return result
Пример #7
0
  def __call__(self):
    '''
    Do the processing.

    :return: The processed data

    '''
    from dials.array_family import flex
    from time import time
    from dials.model.data import make_image
    from libtbx.introspection import machine_memory_info

    # Get the start time
    start_time = time()

    # Set the global process ID
    job.index = self.index

    # Check all reflections have same imageset and get it
    exp_id = list(set(self.reflections['id']))
    imageset = self.experiments[exp_id[0]].imageset
    for i in exp_id[1:]:
      assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"

    # Get the sub imageset
    frame00, frame01 = self.job
    try:
      frame10, frame11 = imageset.get_array_range()
    except Exception:
      frame10, frame11 = (0, len(imageset))
    try:
      assert frame00 < frame01
      assert frame10 < frame11
      assert frame00 >= frame10
      assert frame01 <= frame11
      index0 = frame00 - frame10
      index1 = index0 + (frame01 - frame00)
      assert index0 < index1
      assert index0 >= 0
      assert index1 <= len(imageset)
      imageset = imageset[index0:index1]
    except Exception:
      raise RuntimeError('Programmer Error: bad array range')
    try:
      frame0, frame1 = imageset.get_array_range()
    except Exception:
      frame0, frame1 = (0, len(imageset))

    # Initlize the executor
    self.executor.initialize(frame0, frame1, self.reflections)

    # Set the shoeboxes (dont't allocate)
    self.reflections['shoebox'] = flex.shoebox(
      self.reflections['panel'],
      self.reflections['bbox'],
      allocate=False,
      flatten=self.params.shoebox.flatten)

    # Create the processor
    processor = ShoeboxProcessor(
      self.reflections,
      len(imageset.get_detector()),
      frame0,
      frame1,
      self.params.debug.output)

    # Compute percentage of max available. The function is not portable to
    # windows so need to add a check if the function fails. On windows no
    # warning will be printed
    memory_info = machine_memory_info()
    total_memory = memory_info.memory_total()
    sbox_memory = processor.compute_max_memory_usage()
    if total_memory is not None:
      assert total_memory > 0, "Your system appears to have no memory!"
      assert self.params.block.max_memory_usage >  0.0, "maximum memory usage must be > 0"
      assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
      limit_memory = total_memory * self.params.block.max_memory_usage
      if sbox_memory > limit_memory:
        raise RuntimeError('''
        There was a problem allocating memory for shoeboxes. Possible solutions
        include increasing the percentage of memory allowed for shoeboxes or
        decreasing the block size. This could also be caused by a highly mosaic
        crystal model - is your crystal really this mosaic?
          Total system memory: %g GB
          Limit shoebox memory: %g GB
          Required shoebox memory: %g GB
        ''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9))
      else:
        logger.info(' Memory usage:')
        logger.info('  Total system memory: %g GB' % (total_memory/1e9))
        logger.info('  Limit shoebox memory: %g GB' % (limit_memory/1e9))
        logger.info('  Required shoebox memory: %g GB' % (sbox_memory/1e9))
        logger.info('')

    # Loop through the imageset, extract pixels and process reflections
    read_time = 0.0
    for i in range(len(imageset)):
      st = time()
      image = imageset.get_corrected_data(i)
      mask = imageset.get_mask(i)
      if self.params.lookup.mask is not None:
        assert len(mask) == len(self.params.lookup.mask), \
          "Mask/Image are incorrect size %d %d" % (
            len(mask),
            len(self.params.lookup.mask))
        mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask))

      read_time += time() - st
      processor.next(make_image(image, mask), self.executor)
      del image
      del mask
    assert processor.finished(), "Data processor is not finished"

    # Optionally save the shoeboxes
    if self.params.debug.output and self.params.debug.separate_files:
      output = self.reflections
      if self.params.debug.select is not None:
        output = output.select(self.params.debug.select(output))
      if self.params.debug.split_experiments:
        output = output.split_by_experiment_id()
        for table in output:
          i = table['id'][0]
          table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
      else:
        output.as_pickle('shoeboxes_%d.pickle' % self.index)

    # Delete the shoeboxes
    if self.params.debug.separate_files or not self.params.debug.output:
      del self.reflections['shoebox']

    # Finalize the executor
    self.executor.finalize()

    # Return the result
    result = Result(self.index, self.reflections, self.executor.data())
    result.read_time = read_time
    result.extract_time = processor.extract_time()
    result.process_time = processor.process_time()
    result.total_time = time() - start_time
    return result
Пример #8
0
    def __call__(self):
        """
        Do the processing.

        :return: The processed data
        """
        from dials.array_family import flex
        from dials.model.data import make_image

        # Get the start time
        start_time = time()

        # Set the global process ID
        job.index = self.index

        # Check all reflections have same imageset and get it
        exp_id = list(set(self.reflections["id"]))
        imageset = self.experiments[exp_id[0]].imageset
        for i in exp_id[1:]:
            assert (self.experiments[i].imageset == imageset
                    ), "Task can only handle 1 imageset"

        # Get the sub imageset
        frame00, frame01 = self.job
        try:
            frame10, frame11 = imageset.get_array_range()
        except Exception:
            frame10, frame11 = (0, len(imageset))
        try:
            assert frame00 < frame01
            assert frame10 < frame11
            assert frame00 >= frame10
            assert frame01 <= frame11
            index0 = frame00 - frame10
            index1 = index0 + (frame01 - frame00)
            assert index0 < index1
            assert index0 >= 0
            assert index1 <= len(imageset)
            imageset = imageset[index0:index1]
        except Exception:
            raise RuntimeError("Programmer Error: bad array range")
        try:
            frame0, frame1 = imageset.get_array_range()
        except Exception:
            frame0, frame1 = (0, len(imageset))

        # Initlize the executor
        self.executor.initialize(frame0, frame1, self.reflections)

        # Set the shoeboxes (dont't allocate)
        self.reflections["shoebox"] = flex.shoebox(
            self.reflections["panel"],
            self.reflections["bbox"],
            allocate=False,
            flatten=self.params.shoebox.flatten,
        )

        # Create the processor
        processor = ShoeboxProcessor(
            self.reflections,
            len(imageset.get_detector()),
            frame0,
            frame1,
            self.params.debug.output,
        )

        # Compute expected memory usage and warn if not enough
        total_memory = psutil.virtual_memory().total
        sbox_memory = processor.compute_max_memory_usage()
        assert (self.params.block.max_memory_usage >
                0.0), "maximum memory usage must be > 0"
        assert (self.params.block.max_memory_usage <=
                1.0), "maximum memory usage must be <= 1"
        limit_memory = total_memory * self.params.block.max_memory_usage
        if sbox_memory > limit_memory:
            raise RuntimeError("""
    There was a problem allocating memory for shoeboxes. Possible solutions
    include increasing the percentage of memory allowed for shoeboxes or
    decreasing the block size. This could also be caused by a highly mosaic
    crystal model - is your crystal really this mosaic?
        Total system memory: %g GB
        Limit shoebox memory: %g GB
        Required shoebox memory: %g GB
    """ % (total_memory / 1e9, limit_memory / 1e9, sbox_memory / 1e9))
        else:
            logger.info(" Memory usage:")
            logger.info("  Total system memory: %g GB" % (total_memory / 1e9))
            logger.info("  Limit shoebox memory: %g GB" % (limit_memory / 1e9))
            logger.info("  Required shoebox memory: %g GB" %
                        (sbox_memory / 1e9))
            logger.info("")

        # Loop through the imageset, extract pixels and process reflections
        read_time = 0.0
        for i in range(len(imageset)):
            st = time()
            image = imageset.get_corrected_data(i)
            if imageset.is_marked_for_rejection(i):
                mask = tuple(flex.bool(im.accessor(), False) for im in image)
            else:
                mask = imageset.get_mask(i)
                if self.params.lookup.mask is not None:
                    assert len(mask) == len(self.params.lookup.mask), (
                        "Mask/Image are incorrect size %d %d" %
                        (len(mask), len(self.params.lookup.mask)))
                    mask = tuple(
                        m1 & m2
                        for m1, m2 in zip(self.params.lookup.mask, mask))

            read_time += time() - st
            processor.next(make_image(image, mask), self.executor)
            del image
            del mask
        assert processor.finished(), "Data processor is not finished"

        # Optionally save the shoeboxes
        if self.params.debug.output and self.params.debug.separate_files:
            output = self.reflections
            if self.params.debug.select is not None:
                output = output.select(self.params.debug.select(output))
            if self.params.debug.split_experiments:
                output = output.split_by_experiment_id()
                for table in output:
                    i = table["id"][0]
                    table.as_file("shoeboxes_%d_%d.refl" % (self.index, i))
            else:
                output.as_file("shoeboxes_%d.refl" % self.index)

        # Delete the shoeboxes
        if self.params.debug.separate_files or not self.params.debug.output:
            del self.reflections["shoebox"]

        # Finalize the executor
        self.executor.finalize()

        # Return the result
        result = Result(self.index, self.reflections, self.executor.data())
        result.read_time = read_time
        result.extract_time = processor.extract_time()
        result.process_time = processor.process_time()
        result.total_time = time() - start_time
        return result
Пример #9
0
def integrate_job(block,
                  experiments,
                  reflections,
                  reference,
                  grid_size=5,
                  detector_space=False):
    from dials.algorithms.profile_model.modeller import CircleSampler
    from dials.array_family import flex
    from dials.algorithms.profile_model.gaussian_rs.transform import TransformReverse
    from dials.algorithms.profile_model.gaussian_rs.transform import TransformForward
    from dials.algorithms.profile_model.gaussian_rs.transform import (
        TransformReverseNoModel, )
    from dials.algorithms.profile_model.gaussian_rs.transform import TransformSpec
    from dials.algorithms.profile_model.gaussian_rs import CoordinateSystem
    from dials.algorithms.integration.fit import ProfileFitter
    from dials.array_family import flex
    from dials.model.data import make_image

    reflections["shoebox"] = flex.shoebox(reflections["panel"],
                                          reflections["bbox"],
                                          allocate=True)

    frame0, frame1 = experiments[0].scan.get_array_range()
    frame0 = frame0 + block[0]
    frame1 = frame0 + block[1]

    reflections["shoebox"] = flex.shoebox(reflections["panel"],
                                          reflections["bbox"],
                                          allocate=True)
    extractor = flex.ShoeboxExtractor(reflections, 1, frame0, frame1)

    iset = experiments[0].imageset[block[0]:block[1]]
    for i in range(len(iset)):
        print("Reading image %d" % i)
        data = iset.get_raw_data(i)
        mask = iset.get_mask(i)
        extractor.next(make_image(data, mask))

    print("Computing mask")
    reflections.compute_mask(experiments)

    print("Computing background")
    reflections.compute_background(experiments)

    print("Computing centroid")
    reflections.compute_centroid(experiments)

    print("Computing summed intensity")
    reflections.compute_summed_intensity()

    assert len(reference) % 9 == 0
    num_scan_points = len(reference) // 9

    sampler = CircleSampler(
        experiments[0].detector[0].get_image_size(),
        experiments[0].scan.get_array_range(),
        num_scan_points,
    )

    spec = TransformSpec(
        experiments[0].beam,
        experiments[0].detector,
        experiments[0].goniometer,
        experiments[0].scan,
        experiments[0].profile.sigma_b(deg=False),
        experiments[0].profile.sigma_m(deg=False),
        experiments[0].profile.n_sigma() * 1.5,
        grid_size,
    )

    m2 = experiments[0].goniometer.get_rotation_axis()
    s0 = experiments[0].beam.get_s0()

    Iprf = flex.double(len(reflections))
    Vprf = flex.double(len(reflections))
    Cprf = flex.double(len(reflections))
    Fprf = flex.bool(len(reflections))
    Part = reflections["partiality"]

    reflections["intensity.prf_old.value"] = reflections["intensity.prf.value"]
    reflections["intensity.prf_old.variance"] = reflections[
        "intensity.prf.variance"]

    selection = reflections.get_flags(reflections.flags.integrated_prf)

    reflections.unset_flags(~Fprf, reflections.flags.integrated_prf)

    for i, r in enumerate(reflections):

        if selection[i] == False:
            continue

        s1 = r["s1"]
        phi = r["xyzcal.mm"][2]
        xyz = r["xyzcal.px"]
        bbox = r["bbox"]
        panel = r["panel"]
        image = r["shoebox"].data.as_double()
        background = r["shoebox"].background.as_double()
        mask = r["shoebox"].mask.as_1d(
        ) == 5  # | (r['shoebox'].mask.as_1d() == 3)
        mask.reshape(image.accessor())
        cs = CoordinateSystem(m2, s0, s1, phi)

        index = sampler.nearest(0, xyz)

        profile, profile_mask = reference[index]

        # print flex.sum(profile)
        # print r['partiality']

        if detector_space:

            transform = TransformReverseNoModel(spec, cs, bbox, panel, profile)
            p = transform.profile()
            d = image
            m = mask
            b = background
            # print flex.sum(p)
            Part[i] = flex.sum(p)
            # ysize, xsize = p.all()[1:3]

            # p1 = flex.double(flex.grid(1, ysize , xsize))
            # d1 = flex.double(flex.grid(1, ysize , xsize))
            # b1 = flex.double(flex.grid(1, ysize , xsize))
            # m1 = flex.double(flex.grid(1, ysize , xsize))
            # for k in range(p.all()[0]):
            #   p1 += p[k:k+1,:,:]
            #   d1 += d[k:k+1,:,:]
            #   b1 += b[k:k+1,:,:]
            #   m1 = m[k:k+1,:,:]

            try:

                fit = ProfileFitter(d, b, m, p, 1e-3, 100)
                assert fit.niter() < 100
                Iprf[i] = fit.intensity()
                Vprf[i] = fit.variance()
                Cprf[i] = fit.correlation()
                Fprf[i] = True
                # if r['intensity.sum.value'] > 10 and abs(fit.intensity()) < 1e-3:
                print(
                    r["miller_index"],
                    i,
                    fit.intensity(),
                    r["intensity.sum.value"],
                    r["intensity.prf_old.value"],
                    Part[i],
                    fit.niter(),
                )
                # from matplotlib import pylab
                # pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none')
                # pylab.show()
            except Exception as e:
                print(e)
                pass

        else:

            try:

                transform = TransformForward(spec, cs, bbox, panel, image,
                                             background, mask)

                p = profile
                d = transform.profile()
                b = transform.background()
                m = transform.mask() & profile_mask

                # if r['miller_index'] == (9, -25, 74):
                #   print list(p)
                #   print list(m)
                #   print list(b)
                #   print list(d)
                #   exit(0)

                fit = ProfileFitter(d, b, m, p, 1e-3, 100)
                assert fit.niter() < 100
                Iprf[i] = fit.intensity()[0]
                Vprf[i] = fit.variance()[0]
                Cprf[i] = fit.correlation()
                Fprf[i] = True
                print(r["miller_index"], i, fit.intensity(),
                      r["intensity.prf_old.value"])
                # from matplotlib import pylab
                # pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none')
                # pylab.show()
            except Exception as e:
                pass

    reflections["intensity.prf.value"] = Iprf
    reflections["intensity.prf.variance"] = Vprf
    reflections["intensity.prf.correlation"] = Cprf
    reflections.set_flags(Fprf, reflections.flags.integrated_prf)

    del reflections["shoebox"]

    return reflections
Пример #10
0
  def __call__(self):
    '''
    Do the processing.

    :return: The processed data

    '''
    from dials.model.data import make_image
    from dials.model.data import MultiPanelImageVolume
    from dials.model.data import ImageVolume
    from dials.algorithms.integration.processor import job
    from time import time

    # Set the job index
    job.index = self.index

    # Get the start time
    start_time = time()

    # Check all reflections have same imageset and get it
    exp_id = list(set(self.reflections['id']))
    imageset = self.experiments[exp_id[0]].imageset
    for i in exp_id[1:]:
      assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"

    # Get the sub imageset
    frame00, frame01 = self.frames
    try:
      frame10, frame11 = imageset.get_array_range()
    except Exception:
      frame10, frame11 = (0, len(imageset))
    try:
      assert frame00 < frame01
      assert frame10 < frame11
      assert frame00 >= frame10
      assert frame01 <= frame11
      index0 = frame00 - frame10
      index1 = index0 + (frame01 - frame00)
      assert index0 < index1
      assert index0 >= 0
      assert index1 <= len(imageset)
      imageset = imageset[index0:index1]
    except Exception:
      raise RuntimeError('Programmer Error: bad array range')
    try:
      frame0, frame1 = imageset.get_array_range()
    except Exception:
      frame0, frame1 = (0, len(imageset))

    # Initialise the dataset
    image_volume = MultiPanelImageVolume()
    for panel in self.experiments[0].detector:
      image_volume.add(ImageVolume(
        frame0,
        frame1,
        panel.get_image_size()[1],
        panel.get_image_size()[0]))

    # Read all the images into a block of data
    read_time = 0.0
    for i in range(len(imageset)):
      st = time()
      image = imageset.get_corrected_data(i)
      mask  = imageset.get_mask(i)
      if self.params.integration.lookup.mask is not None:
        assert len(mask) == len(self.params.lookup.mask), \
          "Mask/Image are incorrect size %d %d" % (
            len(mask),
            len(self.params.integration.lookup.mask))
        mask = tuple(m1 & m2 for m1, m2 in zip(self.params.integration.lookup.mask, mask))
      image_volume.set_image(frame0 + i, make_image(image, mask))
      read_time += time() - st
      del image
      del mask

    # Process the data
    st = time()
    data = self.executor.process(
      image_volume,
      self.experiments,
      self.reflections)
    process_time = time() - st

    # Set the result values
    result = Result(self.index, self.reflections)
    result.read_time = read_time
    result.process_time = process_time
    result.total_time = time() - start_time
    result.data = data
    return result