Ejemplo n.º 1
0
def run(args):
  import libtbx.load_env
  usage = "%s [options]" %libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    check_format=False,
    epilog=help_message)

  params, options, args = parser.parse_args(
    show_diff_phil=True, return_unhandled=True)

  results = []
  for mtz in args:
    print mtz
    assert os.path.isfile(mtz), mtz
    results.append(get_merging_stats(
                     mtz, anomalous=params.anomalous,
                     n_bins=params.n_bins,
                     use_internal_variance=params.use_internal_variance,
                     eliminate_sys_absent=params.eliminate_sys_absent))
  plot_merging_stats(results, labels=params.labels,
                     size_inches=params.size_inches,
                     image_dir=params.image_dir)
Ejemplo n.º 2
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  import libtbx.load_env

  usage = "%s [options] image_*.cbf" % (
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_datablocks_from_images=True,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  datablocks = flatten_datablocks(params.input.datablock)

  if len(datablocks) == 0 and len(experiments) == 0 and len(reflections) == 0:
    parser.print_help()
    exit()

  assert(len(datablocks) == 1)

  datablock = datablocks[0]
  imagesets = datablock.extract_imagesets()

  assert(len(imagesets) == 1)

  imageset = imagesets[0]

  images = imageset.indices()
  if params.frames:
    images = params.frames

  d_spacings = []
  intensities = []
  sigmas = []

  for indx in images:
    print 'For frame %d:' % indx
    d, I, sig = background(imageset, indx, n_bins=params.n_bins)

    print '%8s %8s %8s' % ('d', 'I', 'sig')
    for j in range(len(I)):
      print '%8.3f %8.3f %8.3f' % (d[j], I[j], sig[j])

    d_spacings.append(d)
    intensities.append(I)
    sigmas.append(sig)

  if params.plot:
    from matplotlib import pyplot
    fig = pyplot.figure()
    for d, I, sig in zip(d_spacings, intensities, sigmas):
      ds2 = 1/flex.pow2(d)
      pyplot.plot(ds2, I)

    pyplot.show()
Ejemplo n.º 3
0
def run(args):
  from dials.util.options import OptionParser
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_reflections
  import libtbx.load_env

  usage = "%s [options] integrated.pickle experiments.json" % (
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=False)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  if len(experiments) != 1 or len(reflections) != 1:
    parser.print_help()
    exit()

  if not 'shoebox' in reflections[0]:
    print 'Please add shoeboxes to reflection pickle'
    exit()

  results = main(reflections[0], experiments[0], params)

  if results:
    print 'mean result: %.3f' % (sum(results) / len(results))
Ejemplo n.º 4
0
def run(args):
    import libtbx.load_env

    usage = "%s [options] experiment.json indexed.pickle" % libtbx.env.dispatcher_name

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)

    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)
    if len(reflections) == 0 or len(experiments) == 0:
        parser.print_help()
        return
    assert len(reflections) == 1
    assert len(experiments) == 1
    experiment = experiments[0]
    reflections = reflections[0]

    test_P1_crystal_indexing(reflections, experiment, params)
    test_crystal_pointgroup_symmetry(reflections, experiment, params)
Ejemplo n.º 5
0
class Script(object):
  ''' The debugging visualization program. '''

  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # The script usage
    usage  = "usage: %s [options] experiment.json" \
              % libtbx.env.dispatcher_name

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      epilog=help_message,
      read_reflections=True)

  def run(self):


    from dials.util.options import flatten_reflections
    from dials.viewer.viewer_interface import extract_n_show

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)
    table = flatten_reflections(params.input.reflections)
    if len(table) == 0:
      self.parser.print_help()
      return

    extract_n_show(table[0])
Ejemplo n.º 6
0
    def read_experiment_file(self, experiment_file):

        ### open DIALS json file
        phil_scope_str='''
            experiments = 'example_refined_experiments.json'
              '''
        phil_scope = parse(phil_scope_str, process_includes=True)
        parser = OptionParser(
            phil=phil_scope,
            check_format=False,
            read_experiments=True)
        params, options = parser.parse_args(args=[experiment_file], show_diff_phil=True)
        experiments = flatten_experiments(params.input.experiments)
        exp_xtal = experiments.crystals()[0]

        ### define useful attributes
        self.crystal = exp_xtal
        uc = self.crystal.get_unit_cell()
        uc_nospace = str(uc).replace(" ", "")
        uc_nospace_noparen = uc_nospace[1:-1]
        self.unit_cell = uc_nospace_noparen
        self.space_group = self.crystal.get_space_group()
        self.laue_group = self.space_group.laue_group_type()
        # self.a_matrix = crystal.get_A()
        self.experiments = experiments
Ejemplo n.º 7
0
def run(args):
  from dials.util.options import OptionParser
  import libtbx.load_env

  usage = "%s [options] find_spots.json" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    epilog=help_message)

  filenames = [arg for arg in args if os.path.isfile(arg)]
  args = [arg for arg in args if not arg in filenames]

  params, options, args = parser.parse_args(
    show_diff_phil=True, return_unhandled=True)
  if params.nproc is libtbx.Auto:
    from libtbx.introspection import number_of_processors
    params.nproc = number_of_processors(return_value_if_unknown=-1)
  print 'nproc: %i' %params.nproc
  results = work_all(filenames, args, nproc=params.nproc)
  print results

  if params.json is not None:
    import json
    with open(params.json, 'wb') as f:
      json.dump(results, f)
Ejemplo n.º 8
0
def test_function2():
    from dials.util.options import OptionParser
    from time import time
    from omptbx import omp_get_max_threads

    args = [
        "/media/upc86896/Betelgeuse/Data/xia2_test_data/i19_sucrose/processed/dataset1/experiments.json",
        "/media/upc86896/Betelgeuse/Data/xia2_test_data/i19_sucrose/processed/dataset1/shoeboxes_0.pickle",
    ]

    parser = OptionParser(read_experiments=True, read_reflections=True)
    params, options = parser.parse_args(args=args)

    experiment = params.input.experiments[0].data[0]
    reflections = params.input.reflections[0].data

    print "N Threads: ", omp_get_max_threads()
    print "N Refl: ", len(reflections)

    from dials.algorithms.integration.fitrs import test_function

    st = time()
    test_function(
        experiment.beam, experiment.detector, experiment.goniometer, experiment.scan, 0.071016, 0.390601, 5, reflections
    )
    print "Time: ", time() - st
Ejemplo n.º 9
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks

  parser = OptionParser(
    read_datablocks=True,
    read_datablocks_from_images=True,
    phil=phil_scope,
    check_format=True)

  params, options = parser.parse_args(show_diff_phil=True)
  datablocks = flatten_datablocks(params.input.datablock)
  assert len(datablocks) == 1
  imagesets = datablocks[0].extract_imagesets()

  img_count = 0
  import time
  t0 = time.time()
  for imgset in imagesets:
    for i in range(len(imgset)):
      if params.data == 'raw':
        imgset.get_raw_data(i)
      else:
        imgset.get_corrected_data(i)
      img_count += 1
      print "Read %i images" %img_count
  t1 = time.time()
  t = t1 - t0
  print "Read %i images in %.2fs (%.1f images/s)" %(
    img_count, t, img_count/t)

  return
Ejemplo n.º 10
0
def run(args):

    import libtbx.load_env
    from dials.util.options import OptionParser
    from dials.util.options import flatten_reflections
    from libtbx.utils import Sorry
    from libtbx.phil import parse

    phil_scope = parse(
        """
    hklout = hklout.pickle
      .type = str
      .help = "The output pickle file"
  """
    )

    usage = "%s integrated.pickle [hklout=hklout.pickle]" % (libtbx.env.dispatcher_name)

    parser = OptionParser(usage=usage, read_reflections=True, check_format=False, phil=phil_scope)
    params, options = parser.parse_args(show_diff_phil=True)
    reflections = flatten_reflections(params.input.reflections)
    if len(reflections) != 1:
        raise Sorry("exactly 1 reflection table must be specified")

    integrated_data = reflections[0]
    filter_good_reflections(integrated_data).as_pickle(params.hklout)
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_experiments
  from libtbx.utils import Sorry
  import libtbx.load_env

  usage = "%s [options] experiments.json" %libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  experiments = flatten_experiments(params.input.experiments)
  if len(experiments) <= 1:
    parser.print_help()
    return

  hkl = flex.miller_index(params.hkl)

  from dials.algorithms.indexing.compare_orientation_matrices import \
       show_rotation_matrix_differences
  show_rotation_matrix_differences(experiments.crystals(),
                                   miller_indices=hkl)
Ejemplo n.º 12
0
def main():
  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  import libtbx.load_env

  usage = "%s [options] image_*.cbf" % (
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_datablocks_from_images=True,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  datablocks = flatten_datablocks(params.input.datablock)

  if len(datablocks) == 0:
    parser.print_help()
    exit()

  datablock = datablocks[0]
  imageset = datablock.extract_imagesets()[0]
  stability_fft(imageset, params)
Ejemplo n.º 13
0
def run(args):
  from dials.util import log
  import libtbx.load_env
  usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name


  from dials.util.options import OptionParser
  from dials.util.options import flatten_reflections
  from dials.util.options import flatten_experiments
  from dials.array_family import flex

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=False)

  # Configure the logging
  #log.config(info=params.output.log, debug=params.output.debug_log)

  from dials.util.version import dials_version
  logger.info(dials_version())

  # Log the diff phil
  diff_phil = parser.diff_phil.as_str()
  if diff_phil is not '':
    logger.info('The following parameters have been modified:\n')
    logger.info(diff_phil)

  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)
  assert(len(reflections) == 1)
  reflections = reflections[0]

  if len(experiments) == 0:
    parser.print_help()
    return

  #from dials.command_line import refine
  #params = refine.phil_scope.extract()
  indexed_reflections = reflections.select(reflections['id'] > -1)
  from dials.algorithms.refinement import RefinerFactory
  refiner = RefinerFactory.from_parameters_data_experiments(
    params, indexed_reflections, experiments)
  #refiner.run()
  rmsds = refiner.rmsds()
  import math
  xy_rmsds = math.sqrt(rmsds[0]**2 + rmsds[1]**2)

  print rmsds



  return
Ejemplo n.º 14
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_experiments
  import libtbx.load_env

  usage = "%s [options] datablock.json" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_experiments=True,
    check_format=True,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  datablocks = flatten_datablocks(params.input.datablock)
  experiments = flatten_experiments(params.input.experiments)

  if (len(datablocks) == 0 and len(experiments) == 0):
    parser.print_help()
    exit(0)

  if len(datablocks) == 0 and len(experiments) > 0:
    imagesets = experiments.imagesets()
  else:
    imagesets = []
    for datablock in datablocks:
      imagesets.extend(datablock.extract_imagesets())

  assert len(imagesets) == 1
  imageset = imagesets[0]
  gonio = imageset.get_goniometer()
  if not params.detector_distance:
    detector = imageset.get_detector()
    if len(detector) > 1:
      params.detector_distance = detector.hierarchy().get_directed_distance()
    else:
      params.detector_distance = detector[0].get_directed_distance()
  if params.angle:
    assert len(params.angle) == len(gonio.get_angles())
  else:
    for angle in gonio.get_angles():
      params.angle.append(angle)

  import wxtbx.app
  a = wxtbx.app.CCTBXApp(0)
  a.settings = params
  f = ExperimentViewer(
    None, -1, "Experiment viewer", size=(1024,768))
  f.load_imageset(imageset)
  f.Show()
  a.SetTopWindow(f)
  #a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
  a.MainLoop()
Ejemplo n.º 15
0
class Script(object):
  ''' The integration program. '''

  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # The script usage
    usage  = "usage: %s [options] experiment.json" % libtbx.env.dispatcher_name

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      epilog=help_message,
      read_experiments=True)

  def run(self):
    ''' Analyse the background '''
    from dials.util.command_line import heading
    from dials.util.options import flatten_experiments
    from dials.util import log
    from logging import info, debug
    from time import time
    from libtbx.utils import Sorry

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=False)
    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) == 0:
      self.parser.print_help()
      return

    assert len(experiments) == 1

    # Get the imageset
    imageset = experiments[0].imageset

    total_image = None
    total_mask = None
    for i in range(len(imageset)):
      print i
      image = imageset.get_raw_data(i)
      mask = imageset.get_mask(i)
      if total_image is None:
        total_image = image[0]
        total_mask = mask[0]
      else:
        total_image += image[0]
    total_image /= len(imageset)
    print min(total_image)
    print max(total_image)
    print sum(total_image) / len(total_image)

    from matplotlib import pylab
    pylab.imshow(total_image.as_numpy_array(), vmin=0,vmax=2)
    pylab.show()
def run(args):
  import libtbx.load_env
  from libtbx.utils import Sorry
  from dials.util import log
  usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=False)
  datablocks = flatten_datablocks(params.input.datablock)
  reflections = flatten_reflections(params.input.reflections)

  if len(datablocks) == 0 or len(reflections) == 0:
    parser.print_help()
    exit(0)

  # Configure the logging
  log.config(
    info=params.output.log,
    debug=params.output.debug_log)

  # Log the diff phil
  diff_phil = parser.diff_phil.as_str()
  if diff_phil is not '':
    info('The following parameters have been modified:\n')
    info(diff_phil)

  imagesets = []
  for datablock in datablocks:
    imagesets.extend(datablock.extract_imagesets())

  assert len(imagesets) > 0
  assert len(reflections) == len(imagesets)

  if params.scan_range is not None and len(params.scan_range) > 0:
    reflections = [
      filter_reflections_by_scan_range(refl, params.scan_range)
      for refl in reflections]

  dps_params = dps_phil_scope.extract()
  # for development, we want an exhaustive plot of beam probability map:
  dps_params.indexing.plot_search_scope = params.plot_search_scope
  dps_params.indexing.mm_search_scope = params.mm_search_scope

  new_detector, new_beam = discover_better_experimental_model(
    imagesets, reflections, params, dps_params, nproc=params.nproc,
    wide_search_binning=params.wide_search_binning)
  for imageset in imagesets:
    imageset.set_detector(new_detector)
    imageset.set_beam(new_beam)
  from dxtbx.serialize import dump
  dump.datablock(datablock, params.output.datablock)
Ejemplo n.º 17
0
def run(args):
  import libtbx.load_env
  from libtbx.utils import Sorry
  from dials.util import log
  from logging import info
  import cPickle as pickle
  usage = "%s [options] datablock.json strong.pickle" % \
    libtbx.env.dispatcher_name

  # Create the option parser
  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_reflections=True,
    read_datablocks=True,
    check_format=False,
    epilog=help_message)

  # Get the parameters
  params, options = parser.parse_args(show_diff_phil=False)

  # Configure the log
  log.config(
    params.verbosity,
    info='dials.find_hot_pixels.log',
    debug='dials.find_hot_pixels.debug.log')

  # Log the diff phil
  diff_phil = parser.diff_phil.as_str()
  if diff_phil is not '':
    info('The following parameters have been modified:\n')
    info(diff_phil)

  datablocks = flatten_datablocks(params.input.datablock)
  reflections = flatten_reflections(params.input.reflections)

  if len(datablocks) == 0 and len(reflections) == 0:
    parser.print_help()
    exit(0)

  if len(datablocks) > 1:
    raise Sorry("Only one DataBlock can be processed at a time")
  else:
    imagesets = datablocks[0].extract_imagesets()
  if len(reflections) == 0:
    raise Sorry("No reflection lists found in input")
  if len(reflections) > 1:
    raise Sorry("Multiple reflections lists provided in input")

  assert(len(reflections) == 1)
  reflections = reflections[0]

  mask = hot_pixel_mask(imagesets[0], reflections)
  pickle.dump(mask, open(params.output.mask, 'w'), pickle.HIGHEST_PROTOCOL)

  print 'Wrote hot pixel mask to %s' % params.output.mask
  return
Ejemplo n.º 18
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_reflections

  parser = OptionParser(
    phil=master_phil,
    read_datablocks=True,
    read_experiments=True,
    read_reflections=True,
    check_format=False)

  params, options = parser.parse_args(show_diff_phil=True)
  datablocks = flatten_datablocks(params.input.datablock)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)[0]
  if len(params.input.reflections) == 2:
    reflections2 = flatten_reflections(params.input.reflections)[1]
  else:
    reflections2 = None

  # find the reflections in the second set that DO NOT match those in the
  # first set
  mask, _ = reflections2.match_with_reference(reflections)
  reflections2 = reflections2.select(~mask)
  print "{0} reflections from the second set do not match the first". \
    format(len(reflections2))
  #reflections2 = reflections2.select(reflections2["miller_index"] == (-7,2,-25))

  if len(datablocks) == 0:
    if len(experiments) > 0:
      imagesets = experiments.imagesets()
    else:
      parser.print_help()
      return
  elif len(datablocks) > 1:
    raise Sorry("Only one DataBlock can be processed at a time")
  else:
    imagesets = datablocks[0].extract_imagesets()

  if len(imagesets) > 1:
    raise Sorry("Only one ImageSet can be processed at a time")
  imageset = imagesets[0]

  import wxtbx.app
  a = wxtbx.app.CCTBXApp(0)
  a.settings = params
  f = PredRelpViewer(
    None, -1, "Prediction reciprocal lattice viewer", size=(1024,768))
  f.load_reflections2(reflections2)
  f.load_models(imageset, reflections)
  f.Show()
  a.SetTopWindow(f)
  #a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
  a.MainLoop()
Ejemplo n.º 19
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_reflections
  import libtbx.load_env

  usage = "%s [options] datablock.json reflections.pickle" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  datablocks = flatten_datablocks(params.input.datablock)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0:
    parser.print_help()
    exit(0)

  if len(datablocks) == 0 and len(experiments) > 0:
    imagesets = experiments.imagesets()
  else:
    imagesets = []
    for datablock in datablocks:
      imagesets.extend(datablock.extract_imagesets())

  if len(reflections) > 1:
    assert len(reflections) == len(imagesets)
    from scitbx.array_family import flex
    for i in range(len(reflections)):
      reflections[i]['imageset_id'] = flex.int(len(reflections[i]), i)
      if i > 0:
        reflections[0].extend(reflections[i])

  reflections = reflections[0]

  import wxtbx.app
  a = wxtbx.app.CCTBXApp(0)
  a.settings = params
  f = ReciprocalLatticeViewer(
    None, -1, "Reflection data viewer", size=(1024,768))
  f.load_models(imagesets, reflections)
  f.Show()
  a.SetTopWindow(f)
  #a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
  a.MainLoop()
Ejemplo n.º 20
0
class Script(object):
  ''' A class to encapsulate the script. '''

  def __init__(self):
    ''' Initialise the script. '''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # Create the parser
    usage = "usage: %s [options] datablock.json" % libtbx.env.dispatcher_name
    self.parser = OptionParser(
      usage=usage,
      epilog=help_message,
      phil=phil_scope,
      read_datablocks=True)

  def run(self):
    ''' Run the script. '''
    from dials.util.options import flatten_datablocks
    from dxtbx.datablock import DataBlockDumper
    from libtbx.utils import Sorry
    import cPickle as pickle

    # Parse the command line arguments
    params, options = self.parser.parse_args(show_diff_phil=True)
    datablocks = flatten_datablocks(params.input.datablock)

    # Check number of args
    if len(datablocks) == 0:
      self.parser.print_help()
      return

    # Check the mask file is given
    if params.input.mask is None:
      self.parser.print_help()
      return

    # Check nbumber of datablocks
    if len(datablocks) != 1:
      raise Sorry('exactly 1 datablock must be specified')

    # Get the imageset
    datablock = datablocks[0]
    imagesets = datablock.extract_imagesets()
    if len(imagesets) != 1:
      raise Sorry('datablock must contain exactly 1 imageset')
    imageset = imagesets[0]

    # Set the lookup
    imageset.external_lookup.mask.filename = params.input.mask

    # Dump the datablock
    print "Writing datablock to %s" % params.output.datablock
    dump = DataBlockDumper(datablock)
    dump.as_json(filename=params.output.datablock)
Ejemplo n.º 21
0
def run(args):
  import libtbx.load_env
  usage = "%s experiments.json [options]" %libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  experiments = flatten_experiments(params.input.experiments)
  if len(experiments) == 0:
    parser.print_help()
    return
  elif len(experiments) > 1:
    raise Sorry("More than one experiment present")

  assert len(params.miller_index), "Must specify at least one miller_index to predict."

  experiment = experiments[0]

  reflections = flex.reflection_table()
  miller_indices = flex.miller_index()
  entering_flags = flex.bool()
  for mi in params.miller_index:
    miller_indices.append(mi)
    miller_indices.append(mi)
    entering_flags.append(True)
    entering_flags.append(False)
  reflections['miller_index'] = miller_indices
  reflections['entering'] = entering_flags
  reflections['id'] = flex.size_t(len(reflections), 0)

  if params.expand_to_p1:
    from cctbx.miller import expand_to_p1_iselection
    proxy = expand_to_p1_iselection(
      experiment.crystal.get_space_group(),
      anomalous_flag=True,
      indices=miller_indices,
      build_iselection=True)
    reflections = reflections.select(proxy.iselection)
    reflections['miller_index'] = proxy.indices

  from dials.algorithms.refinement.prediction.managed_predictors import ExperimentsPredictor
  predictor = ExperimentsPredictor([experiment])
  predicted = predictor.predict(reflections)

  zmin, zmax = experiment.scan.get_array_range()
  z = predicted['xyzcal.px'].parts()[2]
  predicted = predicted.select((z >= zmin) & (z <= zmax))

  show_predictions(predicted)
Ejemplo n.º 22
0
class Script(object):
  ''' A class to encapsulate the script. '''

  def __init__(self):
    ''' Initialise the script. '''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # The script usage
    usage = "usage: %s [options] /path/to/image/reflection/files" % libtbx.env.dispatcher_name
    self.parser = OptionParser(
      epilog=help_message,
      usage=usage,
      phil=phil_scope,
      read_reflections=True)

  def run(self):
    ''' Run the script. '''
    from dials.array_family import flex
    from dials.util.command_line import Command
    from libtbx.utils import Sorry

    # Parse the command line arguments
    params, options = self.parser.parse_args(show_diff_phil=True)
    if len(params.input.reflections) == 0:
      self.parser.print_help()
      return
    if len(params.input.reflections) <= 1:
      raise Sorry('more than 1 reflection table must be specified')
    tables = [p.data for p in params.input.reflections]

    # Get the number of rows and columns
    nrows = [t.nrows() for t in tables]
    ncols = [t.ncols() for t in tables]

    # Merge the reflection lists
    if params.method == "update":
      assert(all(n == nrows[0] for n in nrows[1:]))
      table = tables[0]
      for t in tables[1:]:
        table.update(t)
    elif params.method == "extend":
      assert(all(n == ncols[0] for n in ncols[1:]))
      table = tables[0]
      for t in tables[1:]:
        table.extend(t)
    else:
      raise RuntimeError('unknown method, %s' % params.method)

    # Write the reflections to the file
    Command.start('Writing %d reflections to %s' % (len(table), params.output))
    table.as_pickle(params.output)
    Command.end('Wrote %d reflections to %s' % (len(table), params.output))
Ejemplo n.º 23
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_reflections
  from dials.util import log

  usage = "%s [options] datablock.json reflections.pickle" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args()
  datablocks = flatten_datablocks(params.input.datablock)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0:
    parser.print_help()
    exit(0)

  ## Configure the logging
  #log.config(info='dials.rl_png.log')

  # Log the diff phil
  diff_phil = parser.diff_phil.as_str()
  if diff_phil is not '':
    logger.info('The following parameters have been modified:\n')
    logger.info(diff_phil)

  reflections = reflections[0]

  if len(datablocks) == 0 and len(experiments) > 0:
    imagesets = experiments.imagesets()
  else:
    imagesets = []
    for datablock in datablocks:
      imagesets.extend(datablock.extract_imagesets())

  f = ReciprocalLatticeJson(settings=params)
  f.load_models(imagesets, reflections)
  f.as_json(filename=params.output.json, compact=params.output.compact)
  print
Ejemplo n.º 24
0
def run(args):

  from dials.util.options import OptionParser
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_experiments
  import libtbx.load_env

  usage = "%s [options] datablock.json | experiments.json" %(
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_datablocks=True,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)
  experiments = flatten_experiments(params.input.experiments)
  datablocks = flatten_datablocks(params.input.datablock)

  if len(experiments) == 0 and len(datablocks) == 0:
    parser.print_help()
    exit(0)

  from dials.command_line.dials_import import ManualGeometryUpdater
  update_geometry = ManualGeometryUpdater(params)


  if len(experiments):
    imagesets = experiments.imagesets()

  elif len(datablocks):

    assert len(datablocks) == 1
    imagesets = datablocks[0].extract_imageset()

  for imageset in imagesets:
    imageset_new = update_geometry(imageset)
    imageset.set_detector(imageset_new.get_detector())
    imageset.set_beam(imageset_new.get_beam())
    imageset.set_goniometer(imageset_new.get_goniometer())
    imageset.set_scan(imageset_new.get_scan())

  from dxtbx.serialize import dump
  if len(experiments):
    print "Saving modified experiments to %s" %params.output.experiments
    dump.experiment_list(experiments, params.output.experiments)
  elif len(datablocks):
    raise NotImplemented
Ejemplo n.º 25
0
    def read_reflection_file(self, reflection_file):

        ### open DIALS pickle file
        phil_scope_str='''
            reflections = 'example_refined.pickle'
              '''
        phil_scope = parse(phil_scope_str, process_includes=True)
        parser = OptionParser(
            phil=phil_scope,
            check_format=False,
            read_reflections=True)
        params, options = parser.parse_args(args=[reflection_file], show_diff_phil=True)
        reflections = flatten_reflections(params.input.reflections)
        self.reflections = reflections
Ejemplo n.º 26
0
def run(args):
  import libtbx.load_env
  from dials.array_family import flex
  from dials.util import log
  from dials.util.version import dials_version

  usage = "%s [options] experiment.json indexed.pickle" % \
    libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_reflections=True,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)

  # Configure the logging
  log.config(info=params.output.log, debug=params.output.debug_log)
  logger.info(dials_version())

  reflections = flatten_reflections(params.input.reflections)
  experiments = flatten_experiments(params.input.experiments)
  if len(reflections) == 0 or len(experiments) == 0:
    parser.print_help()
    return
  assert(len(reflections) == 1)
  assert(len(experiments) == 1)
  experiment = experiments[0]
  reflections = reflections[0]

  # remove reflections with 0, 0, 0 index
  zero = (reflections['miller_index'] == (0, 0, 0))
  logger.info('Removing %d unindexed reflections' % zero.count(True))
  reflections = reflections.select(~zero)

  h, k, l = reflections['miller_index'].as_vec3_double().parts()

  h = h.iround()
  k = k.iround()
  l = l.iround()

  logger.info('Range on h: %d to %d' % (flex.min(h), flex.max(h)))
  logger.info('Range on k: %d to %d' % (flex.min(k), flex.max(k)))
  logger.info('Range on l: %d to %d' % (flex.min(l), flex.max(l)))

  test_P1_crystal_indexing(reflections, experiment, params)
  test_crystal_pointgroup_symmetry(reflections, experiment, params)
Ejemplo n.º 27
0
  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # The phil scope
    phil_scope = parse('''

    print_precision = 4
      .type=int(value_min=0)
      .help="Number of decimal places to print values with"

    ''', process_includes=True)

    # The script usage
    usage  = ("usage: %s [options] [param.phil] experiments1.json "
              "experiments2.json..." % libtbx.env.dispatcher_name)

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      read_experiments=True,
      check_format=False,
      epilog=help_message)
Ejemplo n.º 28
0
  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    from libtbx.phil import parse

    phil_scope = parse('''

      key = 'miller_index'
        .type = str
        .help = "The chosen sort key. This should be a column of "
                "the reflection table."

      reverse = False
        .type = bool
        .help = "Reverse the sort direction"

      output = sorted.pickle
        .type = str
        .help = "The output reflection filename"

    ''')

    # The script usage
    usage  = """
      usage: %s [options] reflections.pickle

    """ % libtbx.env.dispatcher_name

    # Initialise the base class
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      read_reflections=True,
      epilog=help_message)
Ejemplo n.º 29
0
class Script(object):
    def __init__(self):
        # Create the parser
        self.parser = OptionParser(
            read_experiments=True,
            read_datablocks=True,
            read_reflections=True,
            read_datablocks_from_images=True,
            check_format=False,
        )

    def run(self):
        params, options = self.parser.parse_args(show_diff_phil=True)
        datablocks = flatten_datablocks(params.input.datablock)
        experiments = flatten_experiments(params.input.experiments)
        reflections = flatten_reflections(params.input.reflections)
        if len(reflections) > 0:
            reflections = reflections[0]
        else:
            reflections = None

        all_detectors = []
        for db in datablocks:
            all_detectors.extend(db.unique_detectors())

        all_detectors.extend(experiments.detectors())
        display_detectors(all_detectors[: min(len(all_detectors), 10)], reflections=reflections)
Ejemplo n.º 30
0
  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    from libtbx.phil import parse
    import libtbx.load_env

    # The phil scope
    phil_scope = parse('''

      output {
        experiments_prefix = experiments
          .type = str
          .help = "Filename prefix for the split experimental models"

        reflections_prefix = reflections
          .type = str
          .help = "Filename prefix for the split reflections"
      }
    ''', process_includes=True)

    # The script usage
    usage  = "usage: %s [options] [param.phil] " \
             "experiments1.json experiments2.json reflections1.pickle " \
             "reflections2.pickle..." \
             % libtbx.env.dispatcher_name

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      read_reflections=True,
      read_experiments=True,
      check_format=False,
      epilog=help_message)
Ejemplo n.º 31
0
def run():
    """Run the command line filtering script."""

    flags = list(flex.reflection_table.flags.names.items())
    flags.sort(key=itemgetter(0))

    phil_scope = parse(phil_str, process_includes=True)

    # The script usage
    usage = "usage: dials.filter_reflections [options] experiment.expt"

    # Create the parser
    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        epilog=help_message,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
    )

    params, options = parser.parse_args(show_diff_phil=True)
    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)

    log.config(verbosity=options.verbose)

    if not reflections:
        parser.print_help()
        raise Sorry("No valid reflection file given")
    if len(reflections) != 1:
        parser.print_help()
        raise Sorry("Exactly 1 reflection file must be specified")
    reflections = reflections[0]

    # Check if any filter has been set using diff_phil
    filter_def = [
        o for o in parser.diff_phil.objects
        if o.name not in ["input", "output"]
    ]
    if not filter_def:
        print("No filter specified. Performing analysis instead.")
        run_analysis(flags, reflections)
    else:
        run_filtering(params, experiments, reflections)
Ejemplo n.º 32
0
class Script(object):
    def __init__(self):
        '''Initialise the script.'''
        from dials.util.options import OptionParser
        import libtbx.load_env

        # The script usage
        usage  = "usage: %s [options] [param.phil] " \
                 "experiments1.json experiments2.json reflections1.pickle " \
                 "reflections2.pickle..." \
                 % libtbx.env.dispatcher_name

        # Create the parser
        self.parser = OptionParser(usage=usage,
                                   phil=phil_scope,
                                   read_datablocks=True,
                                   epilog=help_message)

    def run(self):
        '''Execute the script.'''

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=True)

        datablocks = flatten_datablocks(params.input.datablock)
        assert len(datablocks) == 1
        imagesets = datablocks[0].extract_imagesets()
        assert len(imagesets) == 1
        imageset = imagesets[0]

        mask = imageset.get_mask(params.image)

        assert (len(mask) == 1)

        print("Num True: %d" % mask[0].count(True))
        print("Num False: %d" % mask[0].count(False))

        from matplotlib import pylab
        pylab.imshow(mask[0].as_numpy_array(), interpolation='none')
        pylab.show()
Ejemplo n.º 33
0
Archivo: test.py Proyecto: monarin/LS49
class Script:
  ''' Class to parse the command line options. '''

  def __init__(self):
    ''' Set the expected options. '''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # Create the option parser
    usage = "usage: %s experiment1.json experiment2.json reflections1.pickle reflections2.pickle" % libtbx.env.dispatcher_name
    self.parser = OptionParser(
      usage=usage,
      sort_options=True,
      phil=phil_scope,
      read_experiments=True,
      read_datablocks=True,
      read_reflections=True,
      check_format=False,
      epilog=help_message)

  def run(self):
    ''' Parse the options. '''


    T1="idx-step5_000000_integrated_experiments.json"

    from dxtbx.model.experiment_list import ExperimentListFactory

    # Parse the command line arguments
    params, options = self.parser.parse_args(show_diff_phil=True)
    self.params = params
    print(params.input.experiments)

    EC = ExperimentListFactory.from_json_file(T1,check_format=False)[0].crystal
    EC.show()
    direct_A = EC.get_A_inverse_as_sqr()
    print(direct_A)
    permute = sqr((0,0,1,0,1,0,-1,0,0))
    sim_compatible = direct_A*permute # permute columns when post multiplying
    print(sim_compatible)
class Script(object):
    def __init__(self):
        # Create the parser
        self.parser = OptionParser(phil=phil_scope, read_experiments=True)

    def run(self):
        params, options = self.parser.parse_args(show_diff_phil=True)
        experiments = flatten_experiments(params.input.experiments)

        detector = experiments[0].detector

        metro = map_detector_to_basis_dict(detector)
        write_cspad_cbf(None,
                        metro,
                        'cbf',
                        None,
                        params.output_def_file,
                        None,
                        detector.hierarchy().get_distance(),
                        header_only=True)

        print "Done"
Ejemplo n.º 35
0
class Script(object):
    def __init__(self):
        """Initialise the script."""
        from dials.util.options import OptionParser
        import libtbx.load_env

        # The script usage
        usage = ("usage: %s [options] [param.phil] "
                 "experiments1.expt experiments2.expt reflections1.refl "
                 "reflections2.refl..." % libtbx.env.dispatcher_name)

        # Create the parser
        self.parser = OptionParser(usage=usage,
                                   phil=phil_scope,
                                   read_experiments=True,
                                   epilog=help_message)

    def run(self):
        """Execute the script."""

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=True)

        experiments = flatten_experiments(params.input.experiments)
        assert len(experiments) == 1
        imageset = experiments[0].imageset

        mask = imageset.get_mask(params.image)

        assert len(mask) == 1

        print("Num True: %d" % mask[0].count(True))
        print("Num False: %d" % mask[0].count(False))

        from matplotlib import pylab

        pylab.imshow(mask[0].as_numpy_array(), interpolation="none")
        pylab.show()
Ejemplo n.º 36
0
class Script(object):
  def __init__(self):
    usage = "whatever"
    self.parser = OptionParser(
         usage=usage,
         phil=phil_scope,
         epilog=help_message,
         check_format=False,
         read_reflections=True,
         read_experiments=True,
         )


  def run(self):
    params, options = self.parser.parse_args(show_diff_phil=False)

    with open(params.input.powder_pattern) as f:
      data = []
      for line in f.readlines():
        x, y = [float(n) for n in line.split()]
        data.append((x, y))

    uc = params.unit_cell
    sg = params.space_group
    d_spacings = []
    mig = miller.index_generator(uc, sg.type(), 0, 0.8*params.d_min)
    for h in mig:
      d_spacings.append(uc.d(h))

    error_cumul = 0

    for x, y in data:
      if x < params.d_min: break
      best_match = min(d_spacings, key=lambda d: abs(x-d))
      error = abs(x-best_match)
      error_cumul += error*y

    print("Cumul. error: ", error_cumul)
Ejemplo n.º 37
0
class InMemScript(DialsProcessScript):
    def __init__(self):
        self.parser = OptionParser(phil=phil_scope)

    def run(self):
        params, options = self.parser.parse_args(show_diff_phil=True)
        assert params.input.single_img is not None

        filebase = os.path.splitext(params.input.single_img)[0]

        for item in dir(params.output):
            value = getattr(params.output, item)
            try:
                if "%s" in value:
                    setattr(params.output, item, value % filebase)
            except Exception:
                pass

        self.params = params
        self.options = options

        # load the image
        img = dxtbx.load(params.input.single_img)
        imgset = MemImageSet([img])
        datablock = DataBlockFactory.from_imageset(imgset)[0]

        # Cannot export MemImageSets
        # if self.params.output.datablock_filename:
        # from dxtbx.datablock import DataBlockDumper
        # dump = DataBlockDumper(datablock)
        # dump.as_json(self.params.output.datablock_filename)

        observed = self.find_spots(datablock)
        experiments, indexed = self.index(datablock, observed)
        experiments = self.refine(experiments, indexed)
        integrated = self.integrate(experiments, indexed)
Ejemplo n.º 38
0
def run(args):
    import libtbx.load_env
    from scitbx import matrix
    from cctbx.sgtbx import lattice_symmetry_group
    from scitbx.math import r3_rotation_axis_and_angle_from_matrix

    usage = "%s [options] experiment_0.expt ..." % libtbx.env.dispatcher_name

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)
    experiments = params.input.experiments

    # check input
    space_group = None
    for experiment in experiments:
        assert len(experiment.data.goniometers()) == 1
        assert len(experiment.data.crystals()) == 1
        crystal = experiment.data.crystals()[0]
        if space_group is None:
            space_group = crystal.get_space_group()
        else:
            assert crystal.get_space_group() == space_group

    reference_U = None
    reference_space_group = None

    for j, experiment in enumerate(experiments):
        goniometer = experiment.data.goniometers()[0]
        F = matrix.sqr(goniometer.get_fixed_rotation())
        crystal = experiment.data.crystals()[0]
        U = matrix.sqr(crystal.get_U())
        B = matrix.sqr(crystal.get_B())
        UB = F * U * B
        UBt = UB.transpose().elems
        a, b, c = matrix.col(UBt[0:3]), matrix.col(UBt[3:6]), matrix.col(
            UBt[6:9])
        axis = matrix.col(goniometer.get_rotation_axis())
        from math import pi

        r2d = 180 / pi
        abc = [a, b, c]
        abc_names = "abc"
        distances = [(r2d * (min(axis.angle(_a), pi - axis.angle(_a))), k)
                     for k, _a in enumerate(abc)]
        close = sorted(distances)[0]
        if reference_U is None:
            reference_U = U
            reference_space_group = lattice_symmetry_group(
                crystal.get_unit_cell(), max_delta=0.0)
            print("%s possible lattice ops" %
                  len(reference_space_group.all_ops()))

        print("Experiment %d" % j)
        print("Closest (original) axis: %s* %.2f" %
              (abc_names[close[1]], close[0]))

        results = []
        for op in reference_space_group.all_ops():
            R = B * matrix.sqr(op.r().as_double()).transpose() * B.inverse()
            relative = (U * R).inverse() * reference_U
            rot = r3_rotation_axis_and_angle_from_matrix(relative)
            results.append((abs(rot.angle()), op.r().as_hkl(), rot))
        results.sort()
        print("Best reindex op for experiment %d: %12s (%.3f)" %
              (j, results[0][1], 180.0 * results[0][2].angle() / pi))

        if results[0][0] > (5 * pi / 180.0):
            print("Rotation: axis: %.4f %.4f %.4f" % results[0][2].axis)
            print("          angle: %.4f degrees" %
                  (180.0 * results[0][2].angle() / pi))
Ejemplo n.º 39
0
def run(args):
    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments

    usage = "dials.shadow_plot [options] models.expt"

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        check_format=True,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)
    experiments = flatten_experiments(params.input.experiments)

    if len(experiments) == 0:
        parser.print_help()
        sys.exit(0)

    assert len(experiments) == 1
    imagesets = experiments.imagesets()

    imageset = imagesets[0]
    goniometer = imageset.get_goniometer()
    detector = imageset.get_detector()
    scan = imageset.get_scan()
    masker = imageset.masker()
    if masker is None:
        raise Sorry("Goniometer model does not support shadowing.")
    angles = goniometer.get_angles()
    names = goniometer.get_names()
    scan_axis = goniometer.get_scan_axis()
    phi = angles[0]

    if params.step_size is libtbx.Auto:
        if params.mode == "1d":
            step = scan.get_oscillation()[1]
        else:
            step = 10
    else:
        step = params.step_size

    if params.mode == "1d":
        if params.oscillation_range is not None:
            start, end = params.oscillation_range
        else:
            start, end = scan.get_oscillation_range()

        scan_points = flex.double(libtbx.utils.frange(start, end, step=step))
        n_px_shadowed = flex.double(scan_points.size(), 0)
        n_px_tot = flex.double(scan_points.size(), 0)

        assert len(angles) == 3
        for i, scan_angle in enumerate(scan_points):
            shadow = masker.project_extrema(detector, scan_angle)
            for p_id in range(len(detector)):
                px_x, px_y = detector[p_id].get_image_size()
                n_px_tot[i] += px_x * px_y
                if shadow[p_id].size() < 4:
                    continue
                n_px_shadowed[i] += polygon_area(shadow[p_id])

    else:
        kappa_values = flex.double(libtbx.utils.frange(0, 360, step=step))
        omega_values = flex.double(libtbx.utils.frange(0, 360, step=step))
        grid = flex.grid(kappa_values.size(), omega_values.size())
        n_px_shadowed = flex.double(grid, 0)
        n_px_tot = flex.double(grid, 0)

        assert len(angles) == 3
        for i, kappa in enumerate(kappa_values):
            for j, omega in enumerate(omega_values):
                masker.set_goniometer_angles((phi, kappa, omega))
                masker.extrema_at_scan_angle(omega)
                shadow = masker.project_extrema(detector, omega)
                for p_id in range(len(detector)):
                    px_x, px_y = detector[p_id].get_image_size()
                    n_px_tot[i, j] += px_x * px_y
                    if shadow[p_id].size() < 4:
                        continue
                    n_px_shadowed[i, j] += polygon_area(shadow[p_id])

    fraction_shadowed = n_px_shadowed / n_px_tot

    if params.output.json is not None:
        if params.mode == "2d":
            raise Sorry("json output not supported for mode=2d")

        print("Writing json output to %s" % params.output.json)
        d = {
            "scan_points": list(scan_points),
            "fraction_shadowed": list(fraction_shadowed),
        }
        with open(params.output.json, "w") as f:
            json.dump(d, f)

    if params.output.plot is not None:
        import matplotlib

        matplotlib.use("Agg")
        from matplotlib import pyplot as plt

        plt.style.use("ggplot")

        if params.mode == "1d":
            plt.plot(scan_points.as_numpy_array(),
                     fraction_shadowed.as_numpy_array() * 100)
            plt.xlabel("%s angle (degrees)" % names[scan_axis])
            plt.ylabel("Shadowed area (%)")
            if params.y_max is not None:
                plt.ylim(0, params.y_max)
            else:
                plt.ylim(0, plt.ylim()[1])
        else:
            fig = plt.imshow(fraction_shadowed.as_numpy_array() * 100,
                             interpolation="bicubic")
            plt.xlabel("%s angle (degrees)" % names[2])
            plt.ylabel("%s angle (degrees)" % names[1])
            plt.xlim(0, 360 / step)
            plt.ylim(0, 360 / step)
            fig.axes.set_xticklabels(
                ["%.0f" % (step * t) for t in plt.xticks()[0]])
            fig.axes.set_yticklabels(
                ["%.0f" % (step * t) for t in plt.yticks()[0]])
            cbar = plt.colorbar()
            cbar.set_label("Shadowed area (%)")

        if params.output.size_inches is not None:
            fig = plt.gcf()
            fig.set_size_inches(params.output.size_inches)
        plt.tight_layout()
        print("Saving plot to %s" % params.output.plot)
        plt.savefig(params.output.plot)
Ejemplo n.º 40
0
    def __init__(self):
        '''Initialise the script.'''
        from dials.util.options import OptionParser
        from libtbx.phil import parse
        import libtbx.load_env

        # The phil scope
        phil_scope = parse('''
      output {
        experiments_filename = refined_experiments.json
          .type = str
          .help = The filename for refined experimental models

        reflections_filename = None
          .type = str
          .help = The filename for output of refined reflections
      }

      n_macrocycles = 1
        .type = int(value_min=1)

      detector_phase {
        include scope dials.algorithms.refinement.refiner.phil_scope
        include scope dials.data.multiprocessing.phil_scope
      }

      crystals_phase {
        include scope dials.algorithms.refinement.refiner.phil_scope
        include scope dials.data.multiprocessing.phil_scope
      }

      reference_detector = *first average
        .type = choice
        .help = First: use the first detector found in the experiment \
                Average: create an average detector from all experiments

    ''',
                           process_includes=True)

        # Set new defaults for detector and crystals refinement phases
        default_phil = parse('''
    crystals_phase.refinement {
        parameterisation {
          beam.fix=all
          detector.fix=all
        }
      reflections.outlier.algorithm=null
      refinery.engine=LevMar
      verbosity=1
    }
    detector_phase.refinement {
        parameterisation {
          beam.fix=all
          crystal.fix=all
          detector.hierarchy_level=1
          sparse=True
        }
      target.gradient_calculation_blocksize=100000
      reflections{
        outlier.algorithm=tukey
        outlier.separate_experiments=False
        weighting_strategy.override=stills
        weighting_strategy.delpsi_constant=1000000
      }
      refinery.engine=LevMar
      verbosity=2
    }
    ''')

        # combine these
        working_phil = phil_scope.fetch(source=default_phil)

        # The script usage
        usage  = "usage: %s [options] [param.phil] " \
                 "experiments.json reflections.pickle" \
                   % libtbx.env.dispatcher_name

        # Create the parser
        self.parser = OptionParser(usage=usage,
                                   phil=working_phil,
                                   read_reflections=True,
                                   read_experiments=True,
                                   check_format=False)
Ejemplo n.º 41
0
class Script(object):
    """ Class to parse the command line options. """
    def __init__(self, phil=phil_scope):
        """ Set the expected options. """
        from dials.util.options import OptionParser

        # Create the option parser
        usage = "usage: %s [options] /path/to/image/files" % libtbx.env.dispatcher_name
        self.parser = OptionParser(
            usage=usage,
            sort_options=True,
            phil=phil,
            read_experiments_from_images=True,
            epilog=help_message,
        )

    def run(self, args=None):
        """ Parse the options. """

        # Parse the command line arguments in two passes to set up logging early
        params, options = self.parser.parse_args(args=args,
                                                 show_diff_phil=False,
                                                 quick_parse=True)

        # Configure logging, if this is the main process
        if __name__ == "__main__":
            from dials.util import log

            log.config(verbosity=options.verbose, logfile=params.output.log)

        from dials.util.version import dials_version

        logger.info(dials_version())

        # Parse the command line arguments completely
        if params.input.ignore_unhandled:
            params, options, unhandled = self.parser.parse_args(
                args=args, show_diff_phil=False, return_unhandled=True)
            # Remove any False values from unhandled (eliminate empty strings)
            unhandled = [x for x in unhandled if x]
        else:
            params, options = self.parser.parse_args(args=args,
                                                     show_diff_phil=False)
            unhandled = None

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil:
            logger.info("The following parameters have been modified:\n")
            logger.info(diff_phil)

        # Print a warning if something unhandled
        if unhandled:
            msg = "Unable to handle the following arguments:\n"
            msg += "\n".join(["  %s" % a for a in unhandled])
            msg += "\n"
            logger.warning(msg)

        # Print help if no input
        if len(params.input.experiments) == 0 and not (params.input.template or
                                                       params.input.directory):
            self.parser.print_help()
            return

        # Setup the experiments importer
        imageset_importer = ImageSetImporter(params)

        # Setup the metadata updater
        metadata_updater = MetaDataUpdater(params)

        # Extract the experiments and loop through
        experiments = metadata_updater(imageset_importer())

        # Compute some numbers
        num_sweeps = 0
        num_stills = 0
        num_images = 0
        for e in experiments:
            if isinstance(e.imageset, ImageSweep):
                num_sweeps += 1
            else:
                num_stills += 1
            num_images += len(e.imageset)
        format_list = {str(e.imageset.get_format_class()) for e in experiments}

        # Print out some bulk info
        logger.info("-" * 80)
        for f in format_list:
            logger.info("  format: %s" % f)
        logger.info("  num images: %d" % num_images)
        logger.info("  num sweeps: %d" % num_sweeps)
        logger.info("  num stills: %d" % num_stills)

        # Print out info for all experiments
        for experiment in experiments:

            # Print some experiment info - override the output of image range
            # if appropriate
            image_range = params.geometry.scan.image_range
            if isinstance(experiment.imageset, ImageSweep):
                imageset_type = "sweep"
            else:
                imageset_type = "stills"

            logger.debug("-" * 80)
            logger.debug("  format: %s" %
                         str(experiment.imageset.get_format_class()))
            logger.debug("  imageset type: %s" % imageset_type)
            if image_range is None:
                logger.debug("  num images:    %d" % len(experiment.imageset))
            else:
                logger.debug("  num images:    %d" %
                             (image_range[1] - image_range[0] + 1))

            logger.debug("")
            logger.debug(experiment.imageset.get_beam())
            logger.debug(experiment.imageset.get_goniometer())
            logger.debug(experiment.imageset.get_detector())
            logger.debug(experiment.imageset.get_scan())

        # Only allow a single sweep
        if params.input.allow_multiple_sweeps is False:
            self.assert_single_sweep(experiments, params)

        # Write the experiments to file
        self.write_experiments(experiments, params)

    def write_experiments(self, experiments, params):
        """
        Output the experiments to file.

        """
        if params.output.experiments:
            logger.info("-" * 80)
            logger.info("Writing experiments to %s" %
                        params.output.experiments)
            experiments.as_file(params.output.experiments,
                                compact=params.output.compact)

    def assert_single_sweep(self, experiments, params):
        """
        Print an error message if more than 1 sweep
        """
        sweeps = [
            e.imageset for e in experiments
            if isinstance(e.imageset, ImageSweep)
        ]

        if len(sweeps) > 1:

            # Print some info about multiple sweeps
            self.diagnose_multiple_sweeps(sweeps, params)

            # Raise exception
            raise Sorry("""
        More than 1 sweep was found. Two things may be happening here:

        1. There really is more than 1 sweep. If you expected this to be the
           case, set the parameter allow_multiple_sweeps=True. If you don't
           expect this, then check the input to dials.import.

        2. There may be something wrong with your image headers (for example,
           the rotation ranges of each image may not match up). You should
           investigate what went wrong, but you can force dials.import to treat
           your images as a single sweep by using the template=image_####.cbf
           parameter (see help).
      """)

    def diagnose_multiple_sweeps(self, sweeps, params):
        """
        Print a diff between sweeps.

        """
        logger.info("")
        for i in range(1, len(sweeps)):
            logger.info("=" * 80)
            logger.info("Diff between sweep %d and %d" % (i - 1, i))
            logger.info("")
            self.print_sweep_diff(sweeps[i - 1], sweeps[i], params)
        logger.info("=" * 80)
        logger.info("")

    def print_sweep_diff(self, sweep1, sweep2, params):
        """
        Print a diff between sweeps.

        """
        from dxtbx.model.experiment_list import SweepDiff

        diff = SweepDiff(params.input.tolerance)
        text = diff(sweep1, sweep2)
        logger.info("\n".join(text))
Ejemplo n.º 42
0
class Script(object):
  '''A class for running the script.'''

  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    from libtbx.phil import parse
    import libtbx.load_env

    # The script usage
    usage = "usage: %s [options] [param.phil] "\
            "{sweep.json | image1.file [image2.file ...]}" \
            % libtbx.env.dispatcher_name

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      epilog=help_message,
      check_format=True,
      read_experiments=True)

  def run(self):
    '''Execute the script.'''
    from dials.util.command_line import Command
    from dials.array_family import flex
    from dials.util.options import flatten_experiments

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)

    # Check the number of experiments
    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) == 0:
      self.parser.print_help()
      return

    predicted_all = flex.reflection_table()

    for i_expt, expt in enumerate(experiments):
      if params.buffer_size > 0:
        # Hack to make the predicter predict reflections outside of the range
        # of the scan
        scan = expt.scan
        image_range = scan.get_image_range()
        oscillation = scan.get_oscillation()
        scan.set_image_range((image_range[0]-params.buffer_size,
                              image_range[1]+params.buffer_size))
        scan.set_oscillation((oscillation[0]-params.buffer_size*oscillation[1],
                              oscillation[1]))

      # Populate the reflection table with predictions
      predicted = flex.reflection_table.from_predictions(
        expt,
        force_static=params.force_static,
        dmin=params.d_min)
      predicted['id'] = flex.int(len(predicted), i_expt)
      predicted_all.extend(predicted)

    # if we are not ignoring shadows, look for reflections in the masked
    # region, see https://github.com/dials/dials/issues/349

    if not params.ignore_shadows:
      from dials.algorithms.shadowing.filter import filter_shadowed_reflections

      shadowed = filter_shadowed_reflections(experiments, predicted_all,
                                             experiment_goniometer=True)
      predicted_all = predicted_all.select(~shadowed)

    try:
      predicted_all.compute_bbox(experiments)
    except Exception:
      pass

    # Save the reflections to file
    Command.start('Saving {0} reflections to {1}'.format(
        len(predicted_all), params.output))
    predicted_all.as_pickle(params.output)
    Command.end('Saved {0} reflections to {1}'.format(
        len(predicted_all), params.output))
Ejemplo n.º 43
0
class Script(object):
    '''A class for running the script.'''
    def __init__(self):
        '''Initialise the script.'''
        from dials.util.options import OptionParser
        from libtbx.phil import parse
        import libtbx.load_env

        # The phil scope
        phil_scope = parse('''
      output {
        experiments_filename = refined_experiments.json
          .type = str
          .help = The filename for refined experimental models

        reflections_filename = None
          .type = str
          .help = The filename for output of refined reflections
      }

      n_macrocycles = 1
        .type = int(value_min=1)

      detector_phase {
        include scope dials.algorithms.refinement.refiner.phil_scope
        include scope dials.data.multiprocessing.phil_scope
      }

      crystals_phase {
        include scope dials.algorithms.refinement.refiner.phil_scope
        include scope dials.data.multiprocessing.phil_scope
      }

      reference_detector = *first average
        .type = choice
        .help = First: use the first detector found in the experiment \
                Average: create an average detector from all experiments

    ''',
                           process_includes=True)

        # Set new defaults for detector and crystals refinement phases
        default_phil = parse('''
    crystals_phase.refinement {
        parameterisation {
          beam.fix=all
          detector.fix=all
        }
      reflections.outlier.algorithm=null
      refinery.engine=LevMar
      verbosity=1
    }
    detector_phase.refinement {
        parameterisation {
          beam.fix=all
          crystal.fix=all
          detector.hierarchy_level=1
          sparse=True
        }
      target.gradient_calculation_blocksize=100000
      reflections{
        outlier.algorithm=tukey
        outlier.separate_experiments=False
        weighting_strategy.override=stills
        weighting_strategy.delpsi_constant=1000000
      }
      refinery.engine=LevMar
      verbosity=2
    }
    ''')

        # combine these
        working_phil = phil_scope.fetch(source=default_phil)

        # The script usage
        usage  = "usage: %s [options] [param.phil] " \
                 "experiments.json reflections.pickle" \
                   % libtbx.env.dispatcher_name

        # Create the parser
        self.parser = OptionParser(usage=usage,
                                   phil=working_phil,
                                   read_reflections=True,
                                   read_experiments=True,
                                   check_format=False)

    def run(self):

        print("Parsing input")
        params, options = self.parser.parse_args(show_diff_phil=True)

        #Configure the logging
        log.config(params.detector_phase.refinement.verbosity,
                   info='dials.refine.log',
                   debug='dials.refine.debug.log')

        # Try to obtain the models and data
        if not params.input.experiments:
            raise Sorry("No Experiments found in the input")
        if not params.input.reflections:
            raise Sorry("No reflection data found in the input")
        try:
            assert len(params.input.reflections) == len(
                params.input.experiments)
        except AssertionError:
            raise Sorry(
                "The number of input reflections files does not match the "
                "number of input experiments")

        # set up global experiments and reflections lists
        from dials.array_family import flex
        reflections = flex.reflection_table()
        global_id = 0
        from dxtbx.model.experiment_list import ExperimentList
        experiments = ExperimentList()

        if params.reference_detector == "first":
            # Use the first experiment of the first experiment list as the reference detector
            ref_exp = params.input.experiments[0].data[0]
        else:
            # Average all the detectors to generate a reference detector
            assert params.detector_phase.refinement.parameterisation.detector.hierarchy_level == 0
            from scitbx.matrix import col
            panel_fasts = []
            panel_slows = []
            panel_oris = []
            for exp_wrapper in params.input.experiments:
                exp = exp_wrapper.data[0]
                if panel_oris:
                    for i, panel in enumerate(exp.detector):
                        panel_fasts[i] += col(panel.get_fast_axis())
                        panel_slows[i] += col(panel.get_slow_axis())
                        panel_oris[i] += col(panel.get_origin())
                else:
                    for i, panel in enumerate(exp.detector):
                        panel_fasts.append(col(panel.get_fast_axis()))
                        panel_slows.append(col(panel.get_slow_axis()))
                        panel_oris.append(col(panel.get_origin()))

            ref_exp = copy.deepcopy(params.input.experiments[0].data[0])
            for i, panel in enumerate(ref_exp.detector):
                # Averaging the fast and slow axes can make them be non-orthagonal. Fix by finding
                # the vector that goes exactly between them and rotate
                # around their cross product 45 degrees from that vector in either direction
                vf = panel_fasts[i] / len(params.input.experiments)
                vs = panel_slows[i] / len(params.input.experiments)
                c = vf.cross(vs)
                angle = vf.angle(vs, deg=True)
                v45 = vf.rotate(c, angle / 2, deg=True)
                vf = v45.rotate(c, -45, deg=True)
                vs = v45.rotate(c, 45, deg=True)
                panel.set_frame(vf, vs,
                                panel_oris[i] / len(params.input.experiments))

            print("Reference detector (averaged):", str(ref_exp.detector))

        # set the experiment factory that combines a crystal with the reference beam
        # and the reference detector
        experiment_from_crystal = ExperimentFromCrystal(
            ref_exp.beam, ref_exp.detector)

        # keep track of the number of refl per accepted experiment for a table
        nrefs_per_exp = []

        # loop through the input, building up the global lists
        for ref_wrapper, exp_wrapper in zip(params.input.reflections,
                                            params.input.experiments):
            refs = ref_wrapper.data
            exps = exp_wrapper.data

            # there might be multiple experiments already here. Loop through them
            for i, exp in enumerate(exps):

                # select the relevant reflections
                sel = refs['id'] == i
                sub_ref = refs.select(sel)

                ## DGW commented out as reflections.minimum_number_of_reflections no longer exists
                #if len(sub_ref) < params.crystals_phase.refinement.reflections.minimum_number_of_reflections:
                #  print "skipping experiment", i, "in", exp_wrapper.filename, "due to insufficient strong reflections in", ref_wrapper.filename
                #  continue

                # build an experiment with this crystal plus the reference models
                combined_exp = experiment_from_crystal(exp.crystal)

                # next experiment ID in series
                exp_id = len(experiments)

                # check this experiment
                if not check_experiment(combined_exp, sub_ref):
                    print("skipping experiment", i, "in", exp_wrapper.filename,
                          "due to poor RMSDs")
                    continue

                # set reflections ID
                sub_ref['id'] = flex.int(len(sub_ref), exp_id)

                # keep number of reflections for the table
                nrefs_per_exp.append(len(sub_ref))

                # obtain mm positions on the reference detector
                sub_ref = indexer_base.map_spots_pixel_to_mm_rad(
                    sub_ref, combined_exp.detector, combined_exp.scan)

                # extend refl and experiments lists
                reflections.extend(sub_ref)
                experiments.append(combined_exp)

        # print number of reflections per accepted experiment
        from libtbx.table_utils import simple_table
        header = ["Experiment", "Nref"]
        rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
        st = simple_table(rows, header)
        print("Number of reflections per experiment")
        print(st.format())

        for cycle in range(params.n_macrocycles):

            print("MACROCYCLE %02d" % (cycle + 1))
            print("=============\n")
            # first run: multi experiment joint refinement of detector with fixed beam and
            # crystals
            print("PHASE 1")

            # SET THIS TEST TO FALSE TO REFINE WHOLE DETECTOR AS SINGLE JOB
            if params.detector_phase.refinement.parameterisation.detector.hierarchy_level > 0:
                experiments = detector_parallel_refiners(
                    params.detector_phase, experiments, reflections)
            else:
                experiments = detector_refiner(params.detector_phase,
                                               experiments, reflections)

            # second run
            print("PHASE 2")
            experiments = crystals_refiner(params.crystals_phase, experiments,
                                           reflections)

        # Save the refined experiments to file
        output_experiments_filename = params.output.experiments_filename
        print('Saving refined experiments to {0}'.format(
            output_experiments_filename))
        from dxtbx.model.experiment_list import ExperimentListDumper
        dump = ExperimentListDumper(experiments)
        dump.as_json(output_experiments_filename)

        # Write out refined reflections, if requested
        if params.output.reflections_filename:
            print('Saving refined reflections to {0}'.format(
                params.output.reflections_filename))
            reflections.as_pickle(params.output.reflections_filename)

        return
Ejemplo n.º 44
0
def run(args):
    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments
    import libtbx.load_env

    usage = "%s [options] models.expt" % (libtbx.env.dispatcher_name)

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)
    experiments = flatten_experiments(params.input.experiments)

    if len(experiments) == 0:
        parser.print_help()
        exit(0)

    expt = experiments[0]

    if params.space_group is not None:
        expt.crystal.set_space_group(params.space_group.group())

    if len(params.align.crystal.vector):
        frame = None
        assert len(params.align.crystal.vector) % 2 == 0
        vectors = []

        name_to_vectors = {
            "a": (a, "direct"),
            "b": (b, "direct"),
            "c": (c, "direct"),
            "a*": (a_star, "reciprocal"),
            "b*": (b_star, "reciprocal"),
            "c*": (c_star, "reciprocal"),
        }

        for v in params.align.crystal.vector:
            v = v.strip()
            if v in name_to_vectors:
                v, frame_ = name_to_vectors[v]
                assert frame is None or frame == frame_
                frame = frame_
            else:
                v = v.replace(",", " ").strip().split()
                assert len(v) == 3
                v = matrix.col([float(v_) for v_ in v])
                if frame is None:
                    frame = params.align.crystal.frame

            vectors.append(v)
        vectors = [(vectors[2 * i], vectors[2 * i + 1])
                   for i in range(len(vectors) // 2)]
    elif params.align.crystal.frame == "direct":
        frame = params.align.crystal.frame
        vectors = ((a, b), (a, c), (b, a), (b, c), (c, a), (c, b))

    else:
        frame = "reciprocal"
        vectors = (
            (a_star, b_star),  # a*, b*
            (a_star, c_star),  # a*, c*
            (b_star, a_star),  # b*, a*
            (b_star, c_star),  # b*, c*
            (c_star, a_star),  # c*, a*
            (c_star, b_star),  # c*, b*
        )

    result = align_crystal(expt, vectors, frame=frame, mode=params.align.mode)
    print(result)
    if params.output.json is not None:
        result.as_json(filename=params.output.json)
Ejemplo n.º 45
0
def run(args):
    from dials.util import log
    usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name

    parser = OptionParser(usage=usage,
                          phil=phil_scope,
                          read_datablocks=True,
                          read_reflections=True,
                          check_format=False,
                          epilog=help_message)

    params, options = parser.parse_args(show_diff_phil=False)
    datablocks = flatten_datablocks(params.input.datablock)
    reflections = flatten_reflections(params.input.reflections)

    if len(datablocks) == 0 or len(reflections) == 0:
        parser.print_help()
        exit(0)

    # Configure the logging
    log.config(info=params.output.log, debug=params.output.debug_log)

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil is not '':
        logger.info('The following parameters have been modified:\n')
        logger.info(diff_phil)

    if params.seed is not None:
        import random
        flex.set_random_seed(params.seed)
        random.seed(params.seed)

    imagesets = []
    for datablock in datablocks:
        imagesets.extend(datablock.extract_imagesets())

    assert len(imagesets) > 0
    assert len(reflections) == len(imagesets)

    if params.scan_range is not None and len(params.scan_range) > 0:
        reflections = [
            filter_reflections_by_scan_range(refl, params.scan_range)
            for refl in reflections
        ]

    dps_params = dps_phil_scope.extract()
    # for development, we want an exhaustive plot of beam probability map:
    dps_params.indexing.plot_search_scope = params.plot_search_scope
    dps_params.indexing.mm_search_scope = params.mm_search_scope

    for i in range(params.n_macro_cycles):
        if params.n_macro_cycles > 1:
            logger.info('Starting macro cycle %i' % (i + 1))
        new_detector, new_beam = discover_better_experimental_model(
            imagesets,
            reflections,
            params,
            dps_params,
            nproc=params.nproc,
            wide_search_binning=params.wide_search_binning)
        for imageset in imagesets:
            imageset.set_detector(new_detector)
            imageset.set_beam(new_beam)
        logger.info('')

    from dxtbx.serialize import dump
    logger.info("Saving optimized datablock to %s" % params.output.datablock)
    dump.datablock(datablock, params.output.datablock)
class Sort(object):
    """A class for running the script."""

    def __init__(self):
        """Initialise the script."""
        from dials.util.options import OptionParser
        from libtbx.phil import parse

        phil_scope = parse(
            """

      key = 'miller_index'
        .type = str
        .help = "The chosen sort key. This should be a column of "
                "the reflection table."

      reverse = False
        .type = bool
        .help = "Reverse the sort direction"

      output = sorted.refl
        .type = str
        .help = "The output reflection filename"

    """
        )

        # The script usage
        usage = (
            """
      usage: %s [options] observations.refl

    """
            % libtbx.env.dispatcher_name
        )

        # Initialise the base class
        self.parser = OptionParser(
            usage=usage, phil=phil_scope, read_reflections=True, epilog=help_message
        )

    @staticmethod
    def sort_permutation(column, reverse=False):
        indices = flex.size_t_range(len(column))
        perm = sorted(indices, key=lambda k: column[k], reverse=reverse)
        return flex.size_t(perm)

    def run(self):
        """Execute the script."""
        from dials.array_family import flex  # noqa: F401, import dependency
        from dials.util.options import flatten_reflections
        from dials.util import Sorry

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=True)
        reflections = flatten_reflections(params.input.reflections)
        if not reflections:
            self.parser.print_help()
            return
        if len(reflections) != 1:
            raise Sorry("exactly 1 reflection table must be specified")
        reflections = reflections[0]

        # Check the key is valid
        assert params.key in reflections

        # Sort the reflections
        print("Sorting by %s with reverse=%r" % (params.key, params.reverse))
        perm = self.sort_permutation(reflections[params.key], params.reverse)
        reflections = reflections.select(perm)

        if options.verbose > 0:
            print("Head of sorted list " + attr + ":")
            n = min(len(reflections), 10)
            for i in range(n):
                print(reflections[i][attr])

        # Save sorted reflections to file
        if params.output:
            print("Saving reflections to {}".format(params.output))
            reflections.as_file(params.output)
Ejemplo n.º 47
0
class Script(object):
    def __init__(self):
        """Initialise the script."""
        # The phil scope
        phil_scope = parse(
            """
      by_detector = False
        .type = bool
        .help = "If True, instead of producing separate files for each"
                "experiment, experiments are grouped by unique detector"
                "model in the input set of experiments. For example, if"
                "there are five detector models in the input data, five"
                "sets of files will be produced, each containing"
                "experiments that reference a single detector model."
      by_wavelength = False
        .type = bool
        .help = "If True, group experiments by wavelength, from low to high"
                "(using a relative tolerance of 1e-4 to match wavelengths)."
      output {
        experiments_prefix = split
          .type = str
          .help = "Filename prefix for the split experimental models"

        reflections_prefix = split
          .type = str
          .help = "Filename prefix for the split reflections"

        template = "{prefix}_{index:0{maxindexlength:d}d}.{extension}"
          .type = str
          .expert_level = 2
          .help = "Template python format string for output filenames."
                  "Replaced variables are prefix (with"
                  "output.{experiments_prefix, reflections_prefix}),"
                  "index (number of split experiment), maxindexlength"
                  "(number of digits of total number of split experiments)"
                  "and extension (default file extension for model and"
                  "reflection files)"

        chunk_size = None
          .type = int
          .expert_level = 2
          .help = "If not None, instead of creating many individual"
                  "files, create composite files with no more than"
                  "chunk_size experiments per file."
        chunk_sizes = None
          .type = ints
          .expert_level = 2
          .help = "If not None, instead of creating many individual"
                  "files, create composite files with the number of"
                  "datasets given in the chunk_sizes list."

      }
    """,
            process_includes=True,
        )

        # The script usage
        usage = (
            "usage: dials.split_experiments [options] [param.phil] "
            "experiments1.expt experiments2.expt reflections1.refl "
            "reflections2.refl..."
        )

        # Create the parser
        self.parser = OptionParser(
            usage=usage,
            phil=phil_scope,
            read_reflections=True,
            read_experiments=True,
            check_format=False,
            epilog=help_message,
        )

    def run(self):
        """Execute the script."""

        # Parse the command line
        params, _ = self.parser.parse_args(show_diff_phil=True)

        # Try to load the models and data
        if not params.input.experiments:
            print("No Experiments found in the input")
            self.parser.print_help()
            return
        if params.input.reflections:
            if len(params.input.reflections) != len(params.input.experiments):
                raise Sorry(
                    "The number of input reflections files does not match the "
                    "number of input experiments"
                )

        experiments = flatten_experiments(params.input.experiments)
        if params.input.reflections:
            reflections = flatten_reflections(params.input.reflections)[0]
        else:
            reflections = None

        experiments_template = functools.partial(
            params.output.template.format,
            prefix=params.output.experiments_prefix,
            maxindexlength=len(str(len(experiments))),
            extension="expt",
        )

        reflections_template = functools.partial(
            params.output.template.format,
            prefix=params.output.reflections_prefix,
            maxindexlength=len(str(len(experiments))),
            extension="refl",
        )

        if params.output.chunk_sizes:
            if not sum(params.output.chunk_sizes) == len(experiments):
                raise Sorry(
                    "Sum of chunk sizes list (%s) not equal to number of experiments (%s)"
                    % (sum(params.output.chunk_sizes), len(experiments))
                )

        if params.by_wavelength:
            if reflections:
                if not reflections.experiment_identifiers():
                    raise Sorry(
                        "Unable to split by wavelength as no experiment "
                        "identifiers are set in the reflection table."
                    )
            if all(experiments.identifiers() == ""):
                raise Sorry(
                    "Unable to split by wavelength as no experiment "
                    "identifiers are set in the experiment list."
                )

            wavelengths = match_wavelengths(experiments)
            for i, wl in enumerate(sorted(wavelengths.keys())):
                expids = []
                new_exps = ExperimentList()
                exp_nos = wavelengths[wl]
                for j in exp_nos:
                    expids.append(experiments[j].identifier)  # string
                    new_exps.append(experiments[j])

                experiment_filename = experiments_template(index=i)
                print(
                    "Saving experiments with wavelength %s to %s"
                    % (wl, experiment_filename)
                )
                new_exps.as_json(experiment_filename)
                if reflections:
                    refls = reflections.select_on_experiment_identifiers(expids)
                    reflections_filename = reflections_template(index=i)
                    print(
                        "Saving reflections with wavelength %s to %s"
                        % (wl, reflections_filename)
                    )
                    refls.as_file(reflections_filename)

        elif params.by_detector:
            assert (
                not params.output.chunk_size
            ), "chunk_size + by_detector is not implemented"
            if reflections is None:
                split_data = {
                    detector: {"experiments": ExperimentList()}
                    for detector in experiments.detectors()
                }
            else:
                split_data = {
                    detector: {
                        "experiments": ExperimentList(),
                        "reflections": flex.reflection_table(),
                    }
                    for detector in experiments.detectors()
                }

            for i, experiment in enumerate(experiments):
                split_expt_id = experiments.detectors().index(experiment.detector)
                experiment_filename = experiments_template(index=split_expt_id)
                print("Adding experiment %d to %s" % (i, experiment_filename))
                split_data[experiment.detector]["experiments"].append(experiment)
                if reflections is not None:
                    reflections_filename = reflections_template(index=split_expt_id)
                    print(
                        "Adding reflections for experiment %d to %s"
                        % (i, reflections_filename)
                    )
                    if reflections.experiment_identifiers().keys():
                        # first find which id value corresponds to experiment in question
                        identifier = experiment.identifier
                        id_ = None
                        for k in reflections.experiment_identifiers().keys():
                            if reflections.experiment_identifiers()[k] == identifier:
                                id_ = k
                                break
                        if id_ is None:
                            raise Sorry(
                                "Unable to find id matching experiment identifier in reflection table."
                            )
                        ref_sel = reflections.select(reflections["id"] == id_)
                        # now reset ids and reset/update identifiers map
                        for k in ref_sel.experiment_identifiers().keys():
                            del ref_sel.experiment_identifiers()[k]
                        new_id = len(split_data[experiment.detector]["experiments"]) - 1
                        ref_sel["id"] = flex.int(len(ref_sel), new_id)
                        ref_sel.experiment_identifiers()[new_id] = identifier
                    else:
                        ref_sel = reflections.select(reflections["id"] == i)
                        ref_sel["id"] = flex.int(
                            len(ref_sel),
                            len(split_data[experiment.detector]["experiments"]) - 1,
                        )
                    split_data[experiment.detector]["reflections"].extend(ref_sel)

            for i, detector in enumerate(experiments.detectors()):
                experiment_filename = experiments_template(index=i)
                print("Saving experiment %d to %s" % (i, experiment_filename))
                split_data[detector]["experiments"].as_json(experiment_filename)

                if reflections is not None:
                    reflections_filename = reflections_template(index=i)
                    print(
                        "Saving reflections for experiment %d to %s"
                        % (i, reflections_filename)
                    )
                    split_data[detector]["reflections"].as_file(reflections_filename)
        elif params.output.chunk_size or params.output.chunk_sizes:

            def save_chunk(chunk_id, expts, refls):
                experiment_filename = experiments_template(index=chunk_id)
                print("Saving chunk %d to %s" % (chunk_id, experiment_filename))
                expts.as_json(experiment_filename)
                if refls is not None:
                    reflections_filename = reflections_template(index=chunk_id)
                    print(
                        "Saving reflections for chunk %d to %s"
                        % (chunk_id, reflections_filename)
                    )
                    refls.as_file(reflections_filename)

            chunk_counter = 0
            chunk_expts = ExperimentList()
            if reflections:
                chunk_refls = flex.reflection_table()
            else:
                chunk_refls = None
            for i, experiment in enumerate(experiments):
                chunk_expts.append(experiment)
                if reflections:
                    if reflections.experiment_identifiers().keys():
                        # first find which id value corresponds to experiment in question
                        identifier = experiment.identifier
                        id_ = None
                        for k in reflections.experiment_identifiers().keys():
                            if reflections.experiment_identifiers()[k] == identifier:
                                id_ = k
                                break
                        if id_ is None:
                            raise Sorry(
                                "Unable to find id matching experiment identifier in reflection table."
                            )
                        ref_sel = reflections.select(reflections["id"] == id_)
                        # now reset ids and reset/update identifiers map
                        for k in ref_sel.experiment_identifiers().keys():
                            del ref_sel.experiment_identifiers()[k]
                        new_id = len(chunk_expts) - 1
                        ref_sel["id"] = flex.int(len(ref_sel), new_id)
                        ref_sel.experiment_identifiers()[new_id] = identifier
                    else:
                        ref_sel = reflections.select(reflections["id"] == i)
                        ref_sel["id"] = flex.int(len(ref_sel), len(chunk_expts) - 1)
                    chunk_refls.extend(ref_sel)
                if params.output.chunk_sizes:
                    chunk_limit = params.output.chunk_sizes[chunk_counter]
                else:
                    chunk_limit = params.output.chunk_size
                if len(chunk_expts) == chunk_limit:
                    save_chunk(chunk_counter, chunk_expts, chunk_refls)
                    chunk_counter += 1
                    chunk_expts = ExperimentList()
                    if reflections:
                        chunk_refls = flex.reflection_table()
                    else:
                        chunk_refls = None
            if len(chunk_expts) > 0:
                save_chunk(chunk_counter, chunk_expts, chunk_refls)
        else:
            for i, experiment in enumerate(experiments):

                experiment_filename = experiments_template(index=i)
                print("Saving experiment %d to %s" % (i, experiment_filename))
                ExperimentList([experiment]).as_json(experiment_filename)

                if reflections is not None:
                    reflections_filename = reflections_template(index=i)
                    print(
                        "Saving reflections for experiment %d to %s"
                        % (i, reflections_filename)
                    )
                    ref_sel = reflections.select(reflections["id"] == i)
                    if ref_sel.experiment_identifiers().keys():
                        identifier = ref_sel.experiment_identifiers()[i]
                        for k in ref_sel.experiment_identifiers().keys():
                            del ref_sel.experiment_identifiers()[k]
                        ref_sel["id"] = flex.int(ref_sel.size(), 0)
                        ref_sel.experiment_identifiers()[0] = identifier
                    else:
                        ref_sel["id"] = flex.int(len(ref_sel), 0)
                    ref_sel.as_file(reflections_filename)

        return
Ejemplo n.º 48
0
class Script:
    def __init__(self):
        from dials.util.options import OptionParser

        self.params = self.parser = None
        if COMM.rank == 0:
            self.parser = OptionParser(
                usage="",  # stage 1 (per-shot) diffBragg refinement",
                sort_options=True,
                phil=phil_scope,
                read_experiments=True,
                read_reflections=True,
                check_format=False,
                epilog="PyCuties")
            self.params, _ = self.parser.parse_args(show_diff_phil=True)
            assert self.params.outdir is not None
        self.params = COMM.bcast(self.params)
        if COMM.rank == 0:
            if not os.path.exists(self.params.outdir):
                utils.safe_makedirs(self.params.outdir)
        COMM.barrier()

        if self.params.logging.logname is None:
            self.params.logging.logname = "main_stage1.log"
        if self.params.profile_name is None:
            self.params.profile_name = "prof_stage1.log"
        from simtbx.diffBragg import mpi_logger
        mpi_logger.setup_logging_from_params(self.params)

    def run(self):
        MAIN_LOGGER = logging.getLogger("diffBragg.main")
        assert os.path.exists(self.params.exp_ref_spec_file)
        input_lines = None
        best_models = None
        if COMM.rank == 0:
            input_lines = open(self.params.exp_ref_spec_file, "r").readlines()
            if self.params.skip is not None:
                input_lines = input_lines[self.params.skip:]
            if self.params.first_n is not None:
                input_lines = input_lines[:self.params.first_n]
            if self.params.sanity_test_input:
                hopper_utils.sanity_test_input_lines(input_lines)

            if self.params.best_pickle is not None:
                logging.info("reading pickle %s" % self.params.best_pickle)
                best_models = pandas.read_pickle(self.params.best_pickle)

            if self.params.dump_gathers:
                if self.params.gathers_dir is None:
                    raise ValueError("Need to provide a file dir path in order to dump_gathers")
                utils.safe_makedirs(self.params.gathers_dir)
        input_lines = COMM.bcast(input_lines)
        best_models = COMM.bcast(best_models)

        if self.params.ignore_existing:
            exp_names_already =None
            if COMM.rank==0:
                exp_names_already = {os.path.basename(f) for f in glob.glob("%s/expers/rank*/*.expt" % self.params.outdir)}
            exp_names_already = COMM.bcast(exp_names_already)

        exp_gatheredRef_spec = []  # optional list of expt, refls, spectra
        for i_exp, line in enumerate(input_lines):
            if i_exp == self.params.max_process:
                break
            if i_exp % COMM.size != COMM.rank:
                continue

            logging.info("COMM.rank %d on shot  %d / %d" % (COMM.rank, i_exp + 1, len(input_lines)))
            line_fields = line.strip().split()
            assert len(line_fields) in [2, 3]
            if len(line_fields) == 2:
                exp, ref = line_fields
                spec = None
            else:
                exp, ref, spec = line_fields

            if self.params.ignore_existing:
                basename = os.path.splitext(os.path.basename(exp))[0]
                opt_exp = "%s_%s_%d.expt" % (self.params.tag, basename, i_exp)
                if opt_exp in exp_names_already:
                    continue

            best = None
            if best_models is not None:
                best = best_models.query("exp_name=='%s'" % exp)
                if len(best) == 0:
                    best = best_models.query("opt_exp_name=='%s'" % exp)
                if len(best) != 1:
                    raise ValueError("Should be 1 entry for exp %s in best pickle %s" % (exp, self.params.best_pickle))
            self.params.simulator.spectrum.filename = spec
            MAIN_LOGGER.info("Modeling %s" % exp)
            Modeler = hopper_utils.DataModeler(self.params)
            if self.params.load_data_from_refls:
                gathered = Modeler.GatherFromReflectionTable(exp, ref)
            else:
                gathered = Modeler.GatherFromExperiment(exp, ref)
            if not gathered:
                logging.warning("No refls in %s; CONTINUE; COMM.rank=%d" % (ref, COMM.rank))
                continue
            if self.params.dump_gathers:
                output_name = os.path.splitext(os.path.basename(exp))[0]
                output_name += "_withData.refl"
                output_name = os.path.join(self.params.gathers_dir, output_name)
                Modeler.dump_gathered_to_refl(output_name, do_xyobs_sanity_check=True)  # NOTE do this is modelin strong spots only
                if self.params.test_gathered_file:
                    all_data = Modeler.all_data.copy()
                    all_roi_id = Modeler.roi_id.copy()
                    all_bg = Modeler.all_background.copy()
                    all_trusted = Modeler.all_trusted.copy()
                    all_pids = np.array(Modeler.pids)
                    all_rois = np.array(Modeler.rois)
                    new_Modeler = hopper_utils.DataModeler(self.params)
                    assert new_Modeler.GatherFromReflectionTable(exp, output_name)
                    assert np.allclose(new_Modeler.all_data, all_data)
                    assert np.allclose(new_Modeler.all_background, all_bg)
                    assert np.allclose(new_Modeler.rois, all_rois)
                    assert np.allclose(new_Modeler.pids, all_pids)
                    assert np.allclose(new_Modeler.all_trusted, all_trusted)
                    assert np.allclose(new_Modeler.roi_id, all_roi_id)

                exp_gatheredRef_spec.append((exp, os.path.abspath(output_name), spec))
                if self.params.only_dump_gathers:
                    continue

            if self.params.refiner.reference_geom is not None:
                detector = ExperimentListFactory.from_json_file(self.params.refiner.reference_geom, check_format=False)[0].detector
                Modeler.E.detector = detector
            Modeler.SimulatorFromExperiment(best)
            Modeler.SIM.D.store_ave_wavelength_image = True
            if self.params.refiner.verbose is not None and COMM.rank==0:
                Modeler.SIM.D.verbose = self.params.refiner.verbose
            if self.params.profile:
                Modeler.SIM.record_timings = True
            if self.params.use_float32:
                Modeler.all_data = Modeler.all_data.astype(np.float32)
                Modeler.all_background = Modeler.all_background.astype(np.float32)

            if self.params.refiner.randomize_devices:
                dev = np.random.choice(self.params.refiner.num_devices)
                logging.info("Rank %d will use randomly chosen device %d on host %s" % (COMM.rank, dev, socket.gethostname()))
            else:
                dev = COMM.rank % self.params.refiner.num_devices
                logging.info("Rank %d will use device %d on host %s" % (COMM.rank, dev, socket.gethostname()))

            Modeler.SIM.D.device_Id = dev

            nparam = len(Modeler.SIM.P)
            x0 = [1] * nparam
            x = Modeler.Minimize(x0)
            if self.params.profile:
                Modeler.SIM.D.show_timings(COMM.rank) #, out)
            save_up(Modeler, x, exp, i_exp, ref)

        if self.params.dump_gathers and self.params.gathered_output_file is not None:
            exp_gatheredRef_spec = COMM.reduce(exp_gatheredRef_spec)
            if COMM.rank == 0:
                o = open(self.params.gathered_output_file, "w")
                for e, r, s in exp_gatheredRef_spec:
                    if s is not None:
                        o.write("%s %s %s\n" % (e,r,s))
                    else:
                        o.write("%s %s\n" % (e,r))
                o.close()
Ejemplo n.º 49
0
if __name__ == "__main__":

    from dials.array_family import flex
    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments
    import libtbx.load_env
    import sys

    # The script usage
    usage = ("usage: %s [options] [param.phil] "
             "{datablock.json | image1.file [image2.file ...]}" %
             libtbx.env.dispatcher_name)

    # Initialise the base class
    parser = OptionParser(usage=usage, phil=phil_scope, read_experiments=True)

    # Parse the command line
    params, options = parser.parse_args(show_diff_phil=False)

    # Ensure we have a data block
    experiments = flatten_experiments(params.input.experiments)
    experiments[0].scan.set_oscillation((0, 1), deg=True)

    # The predicted reflections
    reflections = flex.reflection_table.from_predictions_multi(experiments,
                                                               padding=1)

    # Select only those within 1 deg
    x, y, z = reflections["xyzcal.px"].parts()
    selection = flex.abs(z) < 1
Ejemplo n.º 50
0
class Script(object):
    '''A class for running the script.'''
    def __init__(self):
        '''Initialise the script.'''
        from dials.util.options import OptionParser
        import libtbx.load_env

        # The script usage
        usage  = "usage: %s [options] [param.phil] " \
                 "experiments.json reflections.pickle" \
                   % libtbx.env.dispatcher_name

        # Create the parser
        self.parser = OptionParser(usage=usage,
                                   phil=working_phil,
                                   read_reflections=True,
                                   read_experiments=True,
                                   check_format=False,
                                   epilog=help_message)

    def write_centroids_table(self, refiner, filename):

        matches = refiner.get_matches()

        header = ("H\tK\tL\tFrame_obs\tX_obs\tY_obs\tPhi_obs\tX_calc\t"
                  "Y_calc\tPhi_calc")
        msg_temp = ("%d\t%d\t%d\t%d\t%5.3f\t%5.3f\t%9.6f\t%5.3f\t%5.3f\t%9.6f")
        has_del_psi = 'delpsical.rad' in matches
        if has_del_psi:
            header += "\tDelta_Psi"
            msg_temp += "\t%9.6f"
        header += "\n"
        msg_temp += "\n"

        with open(filename, "w") as f:
            f.write(header)

            for m in matches:
                (h, k, l) = m['miller_index']
                frame = m['xyzobs.px.value'][2]
                x_obs, y_obs, phi_obs = m['xyzobs.mm.value']
                x_calc, y_calc, phi_calc = m['xyzcal.mm']
                if has_del_psi:
                    del_psi = m['delpsical.rad']
                    msg = msg_temp % (h, k, l, frame, x_obs, y_obs, phi_obs,
                                      x_calc, y_calc, phi_calc, del_psi)
                else:
                    msg = msg_temp % (h, k, l, frame, x_obs, y_obs, phi_obs,
                                      x_calc, y_calc, phi_calc)
                f.write(msg)

    @staticmethod
    def check_input(reflections):
        '''Check the input is suitable for refinement. So far just check keys in
    the reflection table. Maybe later check experiments have overlapping models
    etc.'''

        msg = "The supplied reflection table does not have the required data " + \
          "column: {0}"
        for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
            if key not in reflections:
                msg = msg.format(key)
                raise Sorry(msg)

        # FIXME add other things to be checked here
        return

    def run(self):
        '''Execute the script.'''
        from time import time
        import cPickle as pickle
        from dials.util import log
        from dials.algorithms.refinement import RefinerFactory
        from dials.util.options import flatten_reflections, flatten_experiments

        start_time = time()

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=False)
        reflections = flatten_reflections(params.input.reflections)
        experiments = flatten_experiments(params.input.experiments)

        # Try to load the models and data
        nexp = len(experiments)
        if nexp == 0:
            print "No Experiments found in the input"
            self.parser.print_help()
            return
        if len(reflections) == 0:
            print "No reflection data found in the input"
            self.parser.print_help()
            return
        if len(reflections) > 1:
            raise Sorry("Only one reflections list can be imported at present")
        reflections = reflections[0]

        self.check_input(reflections)

        # Configure the logging
        log.config(info=params.output.log, debug=params.output.debug_log)
        from dials.util.version import dials_version
        logger.info(dials_version())

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil is not '':
            logger.info('The following parameters have been modified:\n')
            logger.info(diff_phil)

        # Modify options if necessary
        if params.output.correlation_plot.filename is not None:
            params.refinement.refinery.track_parameter_correlation = True

        # Warn about potentially unhelpful options
        if params.refinement.mp.nproc > 1:
            logger.warning(
                "WARNING: setting nproc > 1 is only helpful in rare "
                "circumstances. It is not recommended for typical data processing "
                "tasks.\n")

        # Get the refiner
        logger.info('Configuring refiner')
        refiner = RefinerFactory.from_parameters_data_experiments(
            params, reflections, experiments)

        # Refine the geometry
        if nexp == 1:
            logger.info('Performing refinement of a single Experiment...')
        else:
            logger.info(
                'Performing refinement of {0} Experiments...'.format(nexp))

        # Refine and get the refinement history
        history = refiner.run()

        if params.output.centroids:
            logger.info("Writing table of centroids to '{0}'".format(
                params.output.centroids))
            self.write_centroids_table(refiner, params.output.centroids)

        # Get the refined experiments
        experiments = refiner.get_experiments()

        # Write scan-varying parameters to file, if there were any
        if params.output.parameter_table:
            scans = experiments.scans()
            if len(scans) > 1:
                logger.info(
                    "Writing a scan-varying parameter table is only supported "
                    "for refinement of a single scan")
            else:
                scan = scans[0]
                text = refiner.get_param_reporter(
                ).varying_params_vs_image_number(scan.get_array_range())
                if text:
                    logger.info(
                        "Writing scan-varying parameter table to {0}".format(
                            params.output.parameter_table))
                    f = open(params.output.parameter_table, "w")
                    f.write(text)
                    f.close()
                else:
                    logger.info("No scan-varying parameter table to write")

        crystals = experiments.crystals()
        if len(crystals) == 1:
            # output the refined model for information
            logger.info('')
            logger.info('Final refined crystal model:')
            logger.info(crystals[0])

        # Save the refined experiments to file
        output_experiments_filename = params.output.experiments
        logger.info('Saving refined experiments to {0}'.format(
            output_experiments_filename))
        from dxtbx.model.experiment_list import ExperimentListDumper
        dump = ExperimentListDumper(experiments)
        dump.as_json(output_experiments_filename)

        # Save reflections with updated predictions if requested (allow to switch
        # this off if it is a time-consuming step)
        if params.output.reflections:
            # Update predictions for all indexed reflections
            logger.info('Updating predictions for indexed reflections')
            preds = refiner.predict_for_indexed()

            # just copy over the columns of interest, leaving behind things
            # added by e.g. scan-varying refinement such as 'block', the
            # U, B and UB matrices and gradients.
            reflections['s1'] = preds['s1']
            reflections['xyzcal.mm'] = preds['xyzcal.mm']
            reflections['xyzcal.px'] = preds['xyzcal.px']
            if 'entering' in preds:
                reflections['entering'] = preds['entering']

            # set used_in_refinement and centroid_outlier flags
            assert len(preds) == len(reflections)
            reflections.unset_flags(
                flex.size_t_range(len(reflections)),
                reflections.flags.used_in_refinement
                | reflections.flags.centroid_outlier)
            mask = preds.get_flags(preds.flags.centroid_outlier)
            reflections.set_flags(mask, reflections.flags.centroid_outlier)
            mask = preds.get_flags(preds.flags.used_in_refinement)
            reflections.set_flags(mask, reflections.flags.used_in_refinement)

            logger.info(
                'Saving reflections with updated predictions to {0}'.format(
                    params.output.reflections))
            if params.output.include_unused_reflections:
                reflections.as_pickle(params.output.reflections)
            else:
                sel = reflections.get_flags(
                    reflections.flags.used_in_refinement)
                reflections.select(sel).as_pickle(params.output.reflections)

        # For debugging, if requested save matches to file
        if params.output.matches:
            matches = refiner.get_matches()
            logger.info(
                'Saving matches (use for debugging purposes) to {0}'.format(
                    params.output.matches))
            matches.as_pickle(params.output.matches)

        # Correlation plot
        if params.output.correlation_plot.filename is not None:
            from os.path import splitext
            root, ext = splitext(params.output.correlation_plot.filename)
            if not ext: ext = ".pdf"

            steps = params.output.correlation_plot.steps
            if steps is None: steps = [history.get_nrows() - 1]

            # extract individual column names or indices
            col_select = params.output.correlation_plot.col_select

            num_plots = 0
            for step in steps:
                fname_base = root
                if len(steps) > 1: fname_base += "_step%02d" % step

                corrmats, labels = refiner.get_parameter_correlation_matrix(
                    step, col_select)
                if [corrmats, labels].count(None) == 0:
                    from dials.algorithms.refinement.refinement_helpers import corrgram
                    for resid_name, corrmat in corrmats.items():
                        plot_fname = fname_base + "_" + resid_name + ext
                        plt = corrgram(corrmat, labels)
                        if plt is not None:
                            logger.info(
                                'Saving parameter correlation plot to {}'.
                                format(plot_fname))
                            plt.savefig(plot_fname)
                            plt.close()
                            num_plots += 1
                    mat_fname = fname_base + ".pickle"
                    with open(mat_fname, 'wb') as handle:
                        for k, corrmat in corrmats.items():
                            corrmats[k] = corrmat.as_scitbx_matrix()
                        logger.info(
                            'Saving parameter correlation matrices to {0}'.
                            format(mat_fname))
                        pickle.dump({
                            'corrmats': corrmats,
                            'labels': labels
                        }, handle)

            if num_plots == 0:
                msg = "Sorry, no parameter correlation plots were produced. Please set " \
                      "track_parameter_correlation=True to ensure correlations are " \
                      "tracked, and make sure correlation_plot.col_select is valid."
                logger.info(msg)

        # Write out refinement history, if requested
        if params.output.history:
            with open(params.output.history, 'wb') as handle:
                logger.info('Saving refinement step history to {0}'.format(
                    params.output.history))
                pickle.dump(history, handle)

        # Log the total time taken
        logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))

        return
Ejemplo n.º 51
0
class Script(object):
  '''A class for running the script.'''

  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    import libtbx.load_env

    # The script usage
    usage = "usage: %s [options] [param.phil] "\
            "datablock.json" \
            % libtbx.env.dispatcher_name

    # Initialise the base class
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      epilog=help_message,
      read_datablocks=True,
      read_reflections=True)

  def run(self):
    '''Execute the script.'''
    from dials.array_family import flex
    from dials.util.options import flatten_datablocks
    from dials.util.options import flatten_reflections
    from time import time
    from dials.util import log
    from libtbx.utils import Sorry
    start_time = time()

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=False)

    # Configure the logging
    log.config(
      params.verbosity,
      info=params.output.log,
      debug=params.output.debug_log)

    from dials.util.version import dials_version
    logger.info(dials_version())

    # Log the diff phil
    diff_phil = self.parser.diff_phil.as_str()
    if diff_phil is not '':
      logger.info('The following parameters have been modified:\n')
      logger.info(diff_phil)

    # Ensure we have a data block
    datablocks = flatten_datablocks(params.input.datablock)
    reflections = flatten_reflections(params.input.reflections)
    if len(datablocks) == 0 and len(reflections) == 0:
      self.parser.print_help()
      return
    elif len(datablocks) != len(reflections):
      raise Sorry("Must have same number of datablocks and reflection tables")

    # Combine the datablocks and reflections
    datablock, reflections = combine(
      datablocks,
      reflections,
      params)

    # Save the reflections to file
    logger.info('\n' + '-' * 80)
    reflections.as_pickle(params.output.reflections)
    logger.info('Saved {0} reflections to {1}'.format(
        len(reflections), params.output.reflections))

    # Save the datablock
    from dxtbx.datablock import DataBlockDumper
    logger.info('Saving datablocks to {0}'.format(
      params.output.datablock))
    dump = DataBlockDumper(datablocks)
    dump.as_file(params.output.datablock)


    # Print the time
    logger.info("Time Taken: %f" % (time() - start_time))
Ejemplo n.º 52
0
class Script(object):
    ''' Class to parse the command line options. '''
    def __init__(self):
        ''' Set the expected options. '''
        from dials.util.options import OptionParser
        import libtbx.load_env

        # Create the option parser
        usage = "usage: %s [options]" % libtbx.env.dispatcher_name
        self.parser = OptionParser(usage=usage,
                                   sort_options=True,
                                   phil=phil_scope,
                                   epilog=help_message)

    def run(self):
        ''' Parse the options. '''
        from dials.util import log
        import libtbx
        from uuid import uuid4
        from dials.util.stream import ZMQStream, Decoder
        from os.path import join, exists
        import os
        import json
        from dxtbx.datablock import DataBlock

        # Parse the command line arguments in two passes to set up logging early
        params, options = self.parser.parse_args(show_diff_phil=False,
                                                 quick_parse=True)

        # Configure logging
        log.config(params.verbosity,
                   info=params.output.log,
                   debug=params.output.debug_log)
        from dials.util.version import dials_version
        logger.info(dials_version())

        # Parse the command line arguments completely
        params, options = self.parser.parse_args(show_diff_phil=False)

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil is not '':
            logger.info('The following parameters have been modified:\n')
            logger.info(diff_phil)

        # Check a stream is given
        if params.input.host is None:
            raise Sorry("An input host needs to be given")

        # Check the directory
        if params.output.directory is None:
            raise Sorry("An output directory needs to be given")
        elif params.output.directory is libtbx.Auto:
            params.output.directory = "/dev/shm/dials-%s" % uuid4()

        # Make the output directory
        if exists(params.output.directory):
            raise Sorry('Directory "%s" already exists' %
                        (params.output.directory))

        # Make the directory
        os.mkdir(params.output.directory)

        # Initialise the stream
        stream = ZMQStream(params.input.host, params.input.port)
        decoder = Decoder(params.output.directory,
                          params.output.image_template)
        imageset = None
        while True:

            # Get the frames from zmq
            frames = stream.receive()

            # Decode the frames
            obj = decoder.decode(frames)

            # Process the object
            if obj.is_header():
                filename = join(params.output.directory, "metadata.json")
                with open(filename, "w") as outfile:
                    json.dump(obj.header, outfile)
                imageset = obj.as_imageset(filename)
                datablocks = [DataBlock([imageset])]
                self.write_datablocks(datablocks, params)
            elif obj.is_image():
                assert imageset is not None
                filename = join(params.output.directory,
                                params.output.image_template % obj.count)
                with open(filename, "wb") as outfile:
                    outfile.write(obj.data)
                filename = join(
                    params.output.directory,
                    "%s.info" % (params.output.image_template % obj.count))
                with open(filename, "w") as outfile:
                    json.dump(obj.info, outfile)
            elif obj.is_endofseries():
                assert imageset is not None
                break
            else:
                raise RuntimeError("Unknown object")

        # Close the stream
        stream.close()

    def write_datablocks(self, datablocks, params):
        '''
    Output the datablock to file.

    '''
        if params.output.datablock:
            logger.info("-" * 80)
            logger.info('Writing datablocks to %s' % params.output.datablock)
            dump = DataBlockDumper(datablocks)
            dump.as_file(params.output.datablock,
                         compact=params.output.compact)
Ejemplo n.º 53
0
def run(args):
    import dials.util.log

    dials.util.log.print_banner()

    from dials.util.options import OptionParser, reflections_and_experiments_from_files

    usage = "dials.show [options] models.expt | image_*.cbf"

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_experiments=True,
        read_experiments_from_images=True,
        read_reflections=True,
        check_format=False,
        epilog=help_message,
    )

    params, options = parser.parse_args(args=args, show_diff_phil=True)
    reflections, experiments = reflections_and_experiments_from_files(
        params.input.reflections, params.input.experiments)

    if len(experiments) == 0 and len(reflections) == 0:
        parser.print_help()
        exit()

    if len(experiments):
        if not all(e.detector for e in experiments):
            sys.exit("Error: experiment has no detector")
        if not all(e.beam for e in experiments):
            sys.exit("Error: experiment has no beam")
        print(
            show_experiments(experiments,
                             show_scan_varying=params.show_scan_varying))

        if params.show_image_statistics:
            print(
                "WARNING: show_image_statistics is deprecated. Please use image_statistics.show_raw or image_statistics.show_corrected instead"
            )
            params.image_statistics.show_raw = True

        if params.image_statistics.show_raw:
            show_image_statistics(experiments, "raw")

        if params.image_statistics.show_corrected:
            show_image_statistics(experiments, "corrected")

        if params.show_shared_models:
            print()
            print(model_connectivity(experiments))

    if len(reflections):
        print(
            show_reflections(
                reflections,
                show_intensities=params.show_intensities,
                show_profile_fit=params.show_profile_fit,
                show_centroids=params.show_centroids,
                show_all_reflection_data=params.show_all_reflection_data,
                show_flags=params.show_flags,
                max_reflections=params.max_reflections,
                show_identifiers=params.show_identifiers,
            ))
Ejemplo n.º 54
0
def run(args):
    import os
    import libtbx.load_env
    from libtbx.utils import Sorry
    from dials.util.options import OptionParser
    usage = "%s [options] datablock.json | image.cbf" % libtbx.env.dispatcher_name

    parser = OptionParser(usage=usage,
                          phil=phil_scope,
                          read_datablocks=True,
                          read_datablocks_from_images=True,
                          check_format=True,
                          epilog=help_message)

    params, options = parser.parse_args(show_diff_phil=True)

    datablocks = flatten_datablocks(params.input.datablock)
    if len(datablocks) == 0:
        parser.print_help()
        exit(0)

    imagesets = datablocks[0].extract_imagesets()

    brightness = params.brightness / 100
    vendortype = "made up"

    # check that binning is a power of 2
    binning = params.binning
    if not (binning > 0 and ((binning & (binning - 1)) == 0)):
        raise Sorry("binning must be a power of 2")

    output_dir = params.output_dir
    if output_dir is None:
        output_dir = "."
    elif not os.path.exists(output_dir):
        os.makedirs(output_dir)

    from rstbx.slip_viewer.tile_generation \
         import _get_flex_image, _get_flex_image_multipanel

    for imageset in imagesets:
        detector = imageset.get_detector()

        if len(detector) > 1:
            raise Sorry(
                'Currently only single panel detectors are supported by %s' %
                libtbx.env.dispatcher_name)

        panel = detector[0]
        scan = imageset.get_scan()
        # XXX is this inclusive or exclusive?
        saturation = panel.get_trusted_range()[1]
        if params.saturation:
            saturation = params.saturation
        if scan is not None and scan.get_oscillation()[1] > 0:
            start, end = scan.get_image_range()
        else:
            start, end = 0, len(imageset)
        for i_image in range(start, end + 1):
            image = imageset.get_raw_data(i_image - start)

            #if len(detector) == 1:
            #image = [image]

            trange = [p.get_trusted_range() for p in detector]
            mask = imageset.get_mask(i_image - start)
            if mask is None:
                mask = [
                    p.get_trusted_range_mask(im)
                    for im, p in zip(image, detector)
                ]

            if params.show_mask:
                for rd, m in zip(image, mask):
                    rd.set_selected(~m, -2)

            image = image_filter(image,
                                 mask,
                                 display=params.display,
                                 gain_value=params.gain,
                                 nsigma_b=params.nsigma_b,
                                 nsigma_s=params.nsigma_s,
                                 global_threshold=params.global_threshold,
                                 min_local=params.min_local,
                                 kernel_size=params.kernel_size)

            show_untrusted = params.show_mask
            if len(detector) > 1:
                # FIXME This doesn't work properly, as flex_image.size2() is incorrect
                # also binning doesn't work
                assert binning == 1
                flex_image = _get_flex_image_multipanel(
                    brightness=brightness,
                    panels=detector,
                    raw_data=image,
                    beam=imageset.get_beam(),
                    show_untrusted=show_untrusted)
            else:
                flex_image = _get_flex_image(brightness=brightness,
                                             data=image[0],
                                             binning=binning,
                                             saturation=saturation,
                                             vendortype=vendortype,
                                             show_untrusted=show_untrusted)

            flex_image.setWindow(0, 0, 1)
            flex_image.adjust(
                color_scheme=colour_schemes.get(params.colour_scheme))

            # now export as a bitmap
            flex_image.prep_string()
            try:
                from PIL import Image
            except ImportError:
                import Image
            # XXX is size//binning safe here?
            try:  # fromstring raises Exception in Pillow >= 3.0.0
                pil_img = Image.fromstring('RGB',
                                           (flex_image.size2() // binning,
                                            flex_image.size1() // binning),
                                           flex_image.export_string)
            except Exception:
                pil_img = Image.frombytes('RGB',
                                          (flex_image.size2() // binning,
                                           flex_image.size1() // binning),
                                          flex_image.export_string)
            path = os.path.join(
                output_dir,
                params.prefix + ("%04d" % i_image) + '.' + params.format)

            print "Exporting %s" % path
            with open(path, 'wb') as tmp_stream:
                pil_img.save(tmp_stream,
                             format=params.format,
                             compress_level=params.png.compress_level,
                             quality=params.jpeg.quality)
Ejemplo n.º 55
0
def run(args):
    from dials.util.options import OptionParser
    import libtbx.load_env

    usage = "%s [options] find_spots.expt" % (libtbx.env.dispatcher_name)

    parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message)

    params, options, args = parser.parse_args(
        show_diff_phil=True, return_unhandled=True
    )

    positions = None
    if params.positions is not None:
        with open(params.positions, "rb") as f:
            positions = flex.vec2_double()
            for line in f.readlines():
                line = (
                    line.replace("(", " ")
                    .replace(")", "")
                    .replace(",", " ")
                    .strip()
                    .split()
                )
                assert len(line) == 3
                i, x, y = [float(l) for l in line]
                positions.append((x, y))

    assert len(args) == 1
    json_file = args[0]
    import json

    with open(json_file, "rb") as f:
        results = json.load(f)

    n_indexed = flex.double()
    fraction_indexed = flex.double()
    n_spots = flex.double()
    n_lattices = flex.double()
    crystals = []
    image_names = flex.std_string()

    for r in results:
        n_spots.append(r["n_spots_total"])
        image_names.append(str(r["image"]))
        if "n_indexed" in r:
            n_indexed.append(r["n_indexed"])
            n_lattices.append(len(r["lattices"]))
            for d in r["lattices"]:
                from dxtbx.model.crystal import CrystalFactory

                crystals.append(CrystalFactory.from_dict(d["crystal"]))
        else:
            n_indexed.append(0)
            n_lattices.append(0)

    if n_indexed.size():
        sel = n_spots > 0
        fraction_indexed = flex.double(n_indexed.size(), 0)
        fraction_indexed.set_selected(sel, n_indexed.select(sel) / n_spots.select(sel))

    import matplotlib

    matplotlib.use("Agg")
    from matplotlib import pyplot

    red = "#e74c3c"

    plot = True
    table = True
    grid = params.grid

    from libtbx import group_args
    from dials.algorithms.spot_finding.per_image_analysis import plot_stats, print_table

    estimated_d_min = flex.double()
    d_min_distl_method_1 = flex.double()
    d_min_distl_method_2 = flex.double()
    n_spots_total = flex.int()
    n_spots_no_ice = flex.int()
    total_intensity = flex.double()

    for d in results:
        estimated_d_min.append(d["estimated_d_min"])
        d_min_distl_method_1.append(d["d_min_distl_method_1"])
        d_min_distl_method_2.append(d["d_min_distl_method_2"])
        n_spots_total.append(d["n_spots_total"])
        n_spots_no_ice.append(d["n_spots_no_ice"])
        total_intensity.append(d["total_intensity"])

    stats = group_args(
        image=image_names,
        n_spots_total=n_spots_total,
        n_spots_no_ice=n_spots_no_ice,
        n_spots_4A=None,
        n_indexed=n_indexed,
        fraction_indexed=fraction_indexed,
        total_intensity=total_intensity,
        estimated_d_min=estimated_d_min,
        d_min_distl_method_1=d_min_distl_method_1,
        d_min_distl_method_2=d_min_distl_method_2,
        noisiness_method_1=None,
        noisiness_method_2=None,
    )

    if plot:
        plot_stats(stats)
        pyplot.clf()
    if table:
        print_table(stats)

    n_rows = 10
    n_rows = min(n_rows, len(n_spots_total))
    perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
    print("Top %i images sorted by number of spots:" % n_rows)
    print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)
    if flex.max(n_indexed) > 0:
        perm_n_indexed = flex.sort_permutation(n_indexed, reverse=True)
        print("Top %i images sorted by number of indexed reflections:" % n_rows)
        print_table(stats, perm=perm_n_indexed, n_rows=n_rows)

    print("Number of indexed lattices: ", (n_indexed > 0).count(True))

    print(
        "Number with valid d_min but failed indexing: ",
        (
            (d_min_distl_method_1 > 0)
            & (d_min_distl_method_2 > 0)
            & (estimated_d_min > 0)
            & (n_indexed == 0)
        ).count(True),
    )

    n_bins = 20
    spot_count_histogram(
        n_spots_total, n_bins=n_bins, filename="hist_n_spots_total.png", log=True
    )
    spot_count_histogram(
        n_spots_no_ice, n_bins=n_bins, filename="hist_n_spots_no_ice.png", log=True
    )
    spot_count_histogram(
        n_indexed.select(n_indexed > 0),
        n_bins=n_bins,
        filename="hist_n_indexed.png",
        log=False,
    )

    if len(crystals):
        plot_unit_cell_histograms(crystals)

    if params.stereographic_projections and len(crystals):
        from dxtbx.model.experiment_list import ExperimentListFactory

        experiments = ExperimentListFactory.from_filenames(
            [image_names[0]], verbose=False
        )
        assert len(experiments) == 1
        imageset = experiments.imagesets()[0]
        s0 = imageset.get_beam().get_s0()
        # XXX what if no goniometer?
        rotation_axis = imageset.get_goniometer().get_rotation_axis()

        indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
        for i, index in enumerate(indices):

            from cctbx import crystal, miller
            from scitbx import matrix

            miller_indices = flex.miller_index([index])
            symmetry = crystal.symmetry(
                unit_cell=crystals[0].get_unit_cell(),
                space_group=crystals[0].get_space_group(),
            )
            miller_set = miller.set(symmetry, miller_indices)
            d_spacings = miller_set.d_spacings()
            d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
            d_spacings = d_spacings.generate_bijvoet_mates()
            miller_indices = d_spacings.indices()

            # plane normal
            d0 = matrix.col(s0).normalize()
            d1 = d0.cross(matrix.col(rotation_axis)).normalize()
            d2 = d1.cross(d0).normalize()
            reference_poles = (d0, d1, d2)

            from dials.command_line.stereographic_projection import (
                stereographic_projection,
            )

            projections = []

            for cryst in crystals:
                reciprocal_space_points = (
                    list(cryst.get_U() * cryst.get_B())
                    * miller_indices.as_vec3_double()
                )
                projections.append(
                    stereographic_projection(reciprocal_space_points, reference_poles)
                )

                # from dials.algorithms.indexing.compare_orientation_matrices import \
                #  difference_rotation_matrix_and_euler_angles
                # R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
                #  crystals[0], cryst)
                # print max(euler_angles)

            from dials.command_line.stereographic_projection import plot_projections

            plot_projections(projections, filename="projections_%s.png" % ("hkl"[i]))
            pyplot.clf()

    def plot_grid(
        values,
        grid,
        file_name,
        cmap=pyplot.cm.Reds,
        vmin=None,
        vmax=None,
        invalid="white",
    ):
        values = values.as_double()
        # At DLS, fast direction appears to be largest direction
        if grid[0] > grid[1]:
            values.reshape(flex.grid(reversed(grid)))
            values = values.matrix_transpose()
        else:
            values.reshape(flex.grid(grid))

        f, ax1 = pyplot.subplots(1)

        mesh1 = ax1.pcolormesh(values.as_numpy_array(), cmap=cmap, vmin=vmin, vmax=vmax)
        mesh1.cmap.set_under(color=invalid, alpha=None)
        mesh1.cmap.set_over(color=invalid, alpha=None)
        ax1.set_aspect("equal")
        ax1.invert_yaxis()
        pyplot.colorbar(mesh1, ax=ax1)
        pyplot.savefig(file_name, dpi=600)
        pyplot.clf()

    def plot_positions(
        values,
        positions,
        file_name,
        cmap=pyplot.cm.Reds,
        vmin=None,
        vmax=None,
        invalid="white",
    ):
        values = values.as_double()
        assert positions.size() >= values.size()
        positions = positions[: values.size()]

        if vmin is None:
            vmin = flex.min(values)
        if vmax is None:
            vmax = flex.max(values)

        x, y = positions.parts()
        dx = flex.abs(x[1:] - x[:-1])
        dy = flex.abs(y[1:] - y[:-1])
        dx = dx.select(dx > 0)
        dy = dy.select(dy > 0)

        scale = 1 / flex.min(dx)
        # print scale
        x = (x * scale).iround()
        y = (y * scale).iround()

        from libtbx.math_utils import iceil

        z = flex.double(flex.grid(iceil(flex.max(y)) + 1, iceil(flex.max(x)) + 1), -2)
        # print z.all()
        for x_, y_, z_ in zip(x, y, values):
            z[y_, x_] = z_

        plot_grid(
            z.as_1d(),
            z.all(),
            file_name,
            cmap=cmap,
            vmin=vmin,
            vmax=vmax,
            invalid=invalid,
        )
        return

    if grid is not None or positions is not None:
        if grid is not None:
            positions = tuple(reversed(grid))
            plotter = plot_grid
        else:
            plotter = plot_positions

        cmap = pyplot.get_cmap(params.cmap)
        plotter(
            n_spots_total,
            positions,
            "grid_spot_count_total.png",
            cmap=cmap,
            invalid=params.invalid,
        )
        plotter(
            n_spots_no_ice,
            positions,
            "grid_spot_count_no_ice.png",
            cmap=cmap,
            invalid=params.invalid,
        )
        plotter(
            total_intensity,
            positions,
            "grid_total_intensity.png",
            cmap=cmap,
            invalid=params.invalid,
        )
        if flex.max(n_indexed) > 0:
            plotter(
                n_indexed,
                positions,
                "grid_n_indexed.png",
                cmap=cmap,
                invalid=params.invalid,
            )
            plotter(
                fraction_indexed,
                positions,
                "grid_fraction_indexed.png",
                cmap=cmap,
                vmin=0,
                vmax=1,
                invalid=params.invalid,
            )

        for i, d_min in enumerate(
            (estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)
        ):
            vmin = flex.min(d_min.select(d_min > 0))
            vmax = flex.max(d_min)
            cmap = pyplot.get_cmap("%s_r" % params.cmap)
            d_min.set_selected(d_min <= 0, vmax)

            if i == 0:
                plotter(
                    d_min,
                    positions,
                    "grid_d_min.png",
                    cmap=cmap,
                    vmin=vmin,
                    vmax=vmax,
                    invalid=params.invalid,
                )
            else:
                plotter(
                    d_min,
                    positions,
                    "grid_d_min_method_%i.png" % i,
                    cmap=cmap,
                    vmin=vmin,
                    vmax=vmax,
                    invalid=params.invalid,
                )

    if flex.max(n_indexed) > 0:
        pyplot.hexbin(n_spots, n_indexed, bins="log", cmap=pyplot.cm.jet, gridsize=50)
        pyplot.colorbar()
        xlim = pyplot.xlim()
        ylim = pyplot.ylim()
        pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
        pyplot.xlim(0, xlim[1])
        pyplot.ylim(0, ylim[1])
        pyplot.xlabel("# spots")
        pyplot.ylabel("# indexed")
        pyplot.savefig("n_spots_vs_n_indexed.png")
        pyplot.clf()

        pyplot.hexbin(
            n_spots, fraction_indexed, bins="log", cmap=pyplot.cm.jet, gridsize=50
        )
        pyplot.colorbar()
        pyplot.xlim(0, pyplot.xlim()[1])
        pyplot.ylim(0, pyplot.ylim()[1])
        pyplot.xlabel("# spots")
        pyplot.ylabel("Fraction indexed")
        pyplot.savefig("n_spots_vs_fraction_indexed.png")
        pyplot.clf()

        pyplot.hexbin(
            n_indexed, fraction_indexed, bins="log", cmap=pyplot.cm.jet, gridsize=50
        )
        pyplot.colorbar()
        pyplot.xlim(0, pyplot.xlim()[1])
        pyplot.ylim(0, pyplot.ylim()[1])
        pyplot.xlabel("# indexed")
        pyplot.ylabel("Fraction indexed")
        pyplot.savefig("n_indexed_vs_fraction_indexed.png")
        pyplot.clf()

        pyplot.hexbin(n_spots, n_lattices, bins="log", cmap=pyplot.cm.jet, gridsize=50)
        pyplot.colorbar()
        pyplot.xlim(0, pyplot.xlim()[1])
        pyplot.ylim(0, pyplot.ylim()[1])
        pyplot.xlabel("# spots")
        pyplot.ylabel("# lattices")
        pyplot.savefig("n_spots_vs_n_lattices.png")
        pyplot.clf()

    pyplot.hexbin(
        estimated_d_min,
        d_min_distl_method_1,
        bins="log",
        cmap=pyplot.cm.jet,
        gridsize=50,
    )
    pyplot.colorbar()
    # pyplot.gca().set_aspect('equal')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    m = max(max(estimated_d_min), max(d_min_distl_method_1))
    pyplot.plot([0, m], [0, m], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel("estimated_d_min")
    pyplot.ylabel("d_min_distl_method_1")
    pyplot.savefig("d_min_vs_distl_method_1.png")
    pyplot.clf()

    pyplot.hexbin(
        estimated_d_min,
        d_min_distl_method_2,
        bins="log",
        cmap=pyplot.cm.jet,
        gridsize=50,
    )
    pyplot.colorbar()
    # pyplot.gca().set_aspect('equal')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    m = max(max(estimated_d_min), max(d_min_distl_method_2))
    pyplot.plot([0, m], [0, m], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel("estimated_d_min")
    pyplot.ylabel("d_min_distl_method_2")
    pyplot.savefig("d_min_vs_distl_method_2.png")
    pyplot.clf()

    pyplot.hexbin(
        d_min_distl_method_1,
        d_min_distl_method_2,
        bins="log",
        cmap=pyplot.cm.jet,
        gridsize=50,
    )
    pyplot.colorbar()
    # pyplot.gca().set_aspect('equal')
    xlim = pyplot.xlim()
    ylim = pyplot.ylim()
    m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
    pyplot.plot([0, m], [0, m], c=red)
    pyplot.xlim(0, xlim[1])
    pyplot.ylim(0, ylim[1])
    pyplot.xlabel("d_min_distl_method_1")
    pyplot.ylabel("d_min_distl_method_2")
    pyplot.savefig("distl_method_1_vs_distl_method_2.png")
    pyplot.clf()

    pyplot.hexbin(n_spots, estimated_d_min, bins="log", cmap=pyplot.cm.jet, gridsize=50)
    pyplot.colorbar()
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel("# spots")
    pyplot.ylabel("estimated_d_min")
    pyplot.savefig("n_spots_vs_d_min.png")
    pyplot.clf()

    pyplot.hexbin(
        n_spots, d_min_distl_method_1, bins="log", cmap=pyplot.cm.jet, gridsize=50
    )
    pyplot.colorbar()
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel("# spots")
    pyplot.ylabel("d_min_distl_method_1")
    pyplot.savefig("n_spots_vs_distl_method_1.png")
    pyplot.clf()

    pyplot.hexbin(
        n_spots, d_min_distl_method_2, bins="log", cmap=pyplot.cm.jet, gridsize=50
    )
    pyplot.colorbar()
    pyplot.xlim(0, pyplot.xlim()[1])
    pyplot.ylim(0, pyplot.ylim()[1])
    pyplot.xlabel("# spots")
    pyplot.ylabel("d_min_distl_method_2")
    pyplot.savefig("n_spots_vs_distl_method_2.png")
    pyplot.clf()
Ejemplo n.º 56
0
class Script(object):
    '''A class for running the script.'''
    def __init__(self):
        self.mpi_helper = mpi_helper()
        self.mpi_logger = mpi_logger()

    def __del__(self):
        self.mpi_helper.finalize()

    def parse_input(self):
        '''Parse input at rank 0 and broadcast the input parameters and options to all ranks'''

        if self.mpi_helper.rank == 0:
            from xfel.merging.application.phil.phil import phil_scope
            help_message = '''Merge xfel data.'''

            # The script usage
            import libtbx.load_env
            self.usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name
            self.parser = None
            '''Initialize the script.'''
            from dials.util.options import OptionParser
            # Create the parser
            self.parser = OptionParser(usage=self.usage,
                                       phil=phil_scope,
                                       epilog=help_message)

            # Parse the command line. quick_parse is required for MPI compatibility
            try:
                bkp = sys.stdout
                sys.stdout = out = StringIO()
                params, options = self.parser.parse_args(show_diff_phil=True,
                                                         quick_parse=True)
                self.mpi_logger.log(out.getvalue())
                if self.mpi_helper.rank == 0:
                    self.mpi_logger.main_log(out.getvalue())
            finally:
                sys.stdout = bkp

            # prepare for transmitting input parameters to all ranks
            self.mpi_logger.log("Broadcasting input parameters...")
            transmitted = dict(params=params, options=options)
        else:
            transmitted = None

        # broadcast parameters and options to all ranks
        self.mpi_logger.log_step_time("BROADCAST_INPUT_PARAMS")

        transmitted = self.mpi_helper.comm.bcast(transmitted, root=0)

        self.params = transmitted['params']
        self.options = transmitted['options']

        self.mpi_logger.set_log_file_paths(self.params)

        self.mpi_logger.log("Received input parameters and options")
        self.mpi_logger.log_step_time("BROADCAST_INPUT_PARAMS", True)

    def run(self):

        import datetime
        time_now = datetime.datetime.now()

        self.mpi_logger.log(str(time_now))
        if self.mpi_helper.rank == 0:
            self.mpi_logger.main_log(str(time_now))

        self.mpi_logger.log_step_time("TOTAL")

        self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS")
        self.parse_input()
        self.mpi_logger.log_step_time("PARSE_INPUT_PARAMS", True)

        # Create the workers using the factories
        self.mpi_logger.log_step_time("CREATE_WORKERS")
        from xfel.merging import application
        import importlib

        workers = []
        steps = default_steps if self.params.dispatch.step_list is None else self.params.dispatch.step_list
        for step in steps:
            step_factory_name = step
            step_additional_info = []

            step_info = step.split(' ')
            assert len(step_info) > 0
            if len(step_info) > 1:
                step_factory_name = step_info[0]
                step_additional_info = step_info[1:]

            factory = importlib.import_module('xfel.merging.application.' +
                                              step_factory_name + '.factory')
            workers.extend(
                factory.factory.from_parameters(self.params,
                                                step_additional_info,
                                                mpi_helper=self.mpi_helper,
                                                mpi_logger=self.mpi_logger))

        # Perform phil validation up front
        for worker in workers:
            worker.validate()
        self.mpi_logger.log_step_time("CREATE_WORKERS", True)

        # Do the work
        experiments = reflections = None
        step = 0
        while (workers):
            worker = workers.pop(0)
            self.mpi_logger.log_step_time("STEP_" + worker.__repr__())
            # Log worker name, i.e. execution step name
            step += 1
            if step > 1:
                self.mpi_logger.log('')
            step_desc = "STEP %d: %s" % (step, worker)
            self.mpi_logger.log(step_desc)

            if self.mpi_helper.rank == 0:
                if step > 1:
                    self.mpi_logger.main_log('')
                self.mpi_logger.main_log(step_desc)

            # Execute worker
            experiments, reflections = worker.run(experiments, reflections)
            self.mpi_logger.log_step_time("STEP_" + worker.__repr__(), True)

        if self.params.output.save_experiments_and_reflections:
            from dxtbx.model.experiment_list import ExperimentListDumper
            import os
            if 'id' not in reflections:
                from dials.array_family import flex
                id_ = flex.int(len(reflections), -1)
                for expt_number, expt in enumerate(experiments):
                    sel = reflections['exp_id'] == expt.identifier
                    id_.set_selected(sel, expt_number)
                reflections['id'] = id_

            reflections.as_pickle(
                os.path.join(
                    self.params.output.output_dir, self.params.output.prefix +
                    "_%06d.pickle" % self.mpi_helper.rank))
            dump = ExperimentListDumper(experiments)
            dump.as_file(
                os.path.join(
                    self.params.output.output_dir, self.params.output.prefix +
                    "_%06d.json" % self.mpi_helper.rank))

        self.mpi_logger.log_step_time("TOTAL", True)
Ejemplo n.º 57
0
class Script(object):
    ''' Class to parse the command line options. '''
    def __init__(self):
        ''' Set the expected options. '''
        from dials.util.options import OptionParser
        import libtbx.load_env

        # Create the option parser
        usage = "usage: %s refined_experiments.json refined_reflections.pickle" % libtbx.env.dispatcher_name
        self.parser = OptionParser(usage=usage,
                                   sort_options=True,
                                   phil=phil_scope,
                                   read_experiments=True,
                                   read_reflections=True,
                                   check_format=False,
                                   epilog=help_message)

    def run(self):
        ''' Parse the options. '''
        from dials.util.options import flatten_experiments, flatten_reflections
        # Parse the command line arguments
        params, options = self.parser.parse_args(show_diff_phil=True)
        experiments = flatten_experiments(params.input.experiments)
        reflections = flatten_reflections(params.input.reflections)
        assert len(reflections) == 1
        reflections = reflections[0]

        domain_size = flex.double()
        mosaic_angle = flex.double()
        filtered_reflections = flex.reflection_table()

        for i in range(len(experiments)):
            refls = reflections.select(reflections['id'] == i)
            try:
                nv = nave_parameters(params=None,
                                     experiments=experiments[i:i + 1],
                                     reflections=refls,
                                     refinery=None,
                                     graph_verbose=False)
                crystal_model_nv = nv()
            except Exception as e:
                continue
            domain_size.append(experiments[i].crystal.get_domain_size_ang() -
                               crystal_model_nv.get_domain_size_ang())
            mosaic_angle.append(
                experiments[i].crystal.get_half_mosaicity_deg() -
                crystal_model_nv.get_half_mosaicity_deg())
            experiments[i].crystal = crystal_model_nv

            refls = refls.select(nv.nv_acceptance_flags)
            filtered_reflections.extend(refls)

        print "Saving new experiments as %s" % params.output.experiments
        dump = ExperimentListDumper(experiments)
        dump.as_json(params.output.experiments)

        print "Removed %d out of %d reflections as outliers" % (
            len(reflections) - len(filtered_reflections), len(reflections))
        print "Saving filtered reflections as %s" % params.output.experiments
        filtered_reflections.as_pickle(params.output.reflections)

        if params.plot_changes:
            domain_size = domain_size.select((domain_size >= -10)
                                             & (domain_size <= 10))
            mosaic_angle = mosaic_angle.select((mosaic_angle >= -0.1)
                                               & (mosaic_angle <= 0.1))

            for d in [domain_size, mosaic_angle]:
                f = plt.figure()
                plt.hist(d, bins=30)
            plt.show()
Ejemplo n.º 58
0
class Script(object):
    """ Script to process dump XFEL data at LCLS """
    def __init__(self):
        """ Set up the option parser. Arguments come from the command line or a phil file """
        self.usage = """
%s input.experiment=experimentname input.run_num=N input.address=address
 format.file_format=cbf format.cbf.detz_offset=N
%s input.experiment=experimentname input.run_num=N input.address=address
 format.file_format=pickle format.pickle.cfg=path
    """ % (libtbx.env.dispatcher_name, libtbx.env.dispatcher_name)

        self.parser = OptionParser(usage=self.usage, phil=phil_scope)

    def run(self):
        """ Process all images assigned to this thread """
        params, options = self.parser.parse_args(show_diff_phil=True)

        if params.input.experiment is None or \
           params.input.run_num is None or \
           params.input.address is None:
            raise Usage(self.usage)

        if params.format.file_format == "cbf":
            if params.format.cbf.detz_offset is None:
                raise Usage(self.usage)
        elif params.format.file_format == "pickle":
            if params.format.pickle.cfg is None:
                raise Usage(self.usage)
        else:
            raise Usage(self.usage)

        if not os.path.exists(params.output.output_dir):
            raise Sorry("Output path not found:" + params.output.output_dir)

        # Save the paramters
        self.params = params
        self.options = options

        from mpi4py import MPI
        comm = MPI.COMM_WORLD
        rank = comm.Get_rank(
        )  # each process in MPI has a unique id, 0-indexed
        size = comm.Get_size()  # size: number of processes running in this job

        # set up psana
        if params.format.file_format == "pickle":
            psana.setConfigFile(params.format.pickle.cfg)

        dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment,
                                              params.input.run_num)
        ds = psana.DataSource(dataset_name)

        if params.format.file_format == "cbf":
            src = psana.Source('DetInfo(%s)' % params.input.address)
            psana_det = psana.Detector(params.input.address, ds.env())

        # set this to sys.maxint to analyze all events
        if params.dispatch.max_events is None:
            max_events = sys.maxint
        else:
            max_events = params.dispatch.max_events

        for run in ds.runs():
            if params.format.file_format == "cbf":
                # load a header only cspad cbf from the slac metrology
                base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
                    run, params.input.address)
                if base_dxtbx is None:
                    raise Sorry("Couldn't load calibration file for run %d" %
                                run.run())

            # list of all events
            times = run.times()
            nevents = min(len(times), max_events)
            # chop the list into pieces, depending on rank.  This assigns each process
            # events such that the get every Nth event where N is the number of processes
            mytimes = [
                times[i] for i in xrange(nevents) if (i + rank) % size == 0
            ]

            for i in xrange(len(mytimes)):
                evt = run.event(mytimes[i])
                id = evt.get(psana.EventId)
                print "Event #", i, " has id:", id

                timestamp = cspad_tbx.evt_timestamp(
                    cspad_tbx.evt_time(evt))  # human readable format
                if timestamp is None:
                    print "No timestamp, skipping shot"
                    continue
                t = timestamp
                s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[
                    17:19] + t[20:23]
                print "Processing shot", s

                if params.format.file_format == "pickle":
                    if evt.get("skip_event"):
                        print "Skipping event", id
                        continue
                    # the data needs to have already been processed and put into the event by psana
                    data = evt.get(params.format.pickle.out_key)
                    if data is None:
                        print "No data"
                        continue

                    # set output paths according to the templates
                    path = os.path.join(params.output.output_dir,
                                        "shot-" + s + ".pickle")

                    print "Saving", path
                    easy_pickle.dump(path, data)

                elif params.format.file_format == "cbf":
                    # get numpy array, 32x185x388
                    data = cspad_cbf_tbx.get_psana_corrected_data(
                        psana_det,
                        evt,
                        use_default=False,
                        dark=True,
                        common_mode=None,
                        apply_gain_mask=params.format.cbf.gain_mask_value
                        is not None,
                        gain_mask_value=params.format.cbf.gain_mask_value,
                        per_pixel_gain=False)

                    distance = cspad_tbx.env_distance(
                        params.input.address, run.env(),
                        params.format.cbf.detz_offset)
                    if distance is None:
                        print "No distance, skipping shot"
                        continue

                    if self.params.format.cbf.override_energy is None:
                        wavelength = cspad_tbx.evt_wavelength(evt)
                        if wavelength is None:
                            print "No wavelength, skipping shot"
                            continue
                    else:
                        wavelength = 12398.4187 / self.params.format.cbf.override_energy

                    # stitch together the header, data and metadata into the final dxtbx format object
                    cspad_img = cspad_cbf_tbx.format_object_from_data(
                        base_dxtbx, data, distance, wavelength, timestamp,
                        params.input.address)
                    path = os.path.join(params.output.output_dir,
                                        "shot-" + s + ".cbf")
                    print "Saving", path

                    # write the file
                    import pycbf
                    cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
                      pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)

            run.end()
        ds.end()
Ejemplo n.º 59
0
    def __init__(self):
        """Initialise the script."""
        # The phil scope
        phil_scope = parse(
            """
      by_detector = False
        .type = bool
        .help = "If True, instead of producing separate files for each"
                "experiment, experiments are grouped by unique detector"
                "model in the input set of experiments. For example, if"
                "there are five detector models in the input data, five"
                "sets of files will be produced, each containing"
                "experiments that reference a single detector model."
      by_wavelength = False
        .type = bool
        .help = "If True, group experiments by wavelength, from low to high"
                "(using a relative tolerance of 1e-4 to match wavelengths)."
      output {
        experiments_prefix = split
          .type = str
          .help = "Filename prefix for the split experimental models"

        reflections_prefix = split
          .type = str
          .help = "Filename prefix for the split reflections"

        template = "{prefix}_{index:0{maxindexlength:d}d}.{extension}"
          .type = str
          .expert_level = 2
          .help = "Template python format string for output filenames."
                  "Replaced variables are prefix (with"
                  "output.{experiments_prefix, reflections_prefix}),"
                  "index (number of split experiment), maxindexlength"
                  "(number of digits of total number of split experiments)"
                  "and extension (default file extension for model and"
                  "reflection files)"

        chunk_size = None
          .type = int
          .expert_level = 2
          .help = "If not None, instead of creating many individual"
                  "files, create composite files with no more than"
                  "chunk_size experiments per file."
        chunk_sizes = None
          .type = ints
          .expert_level = 2
          .help = "If not None, instead of creating many individual"
                  "files, create composite files with the number of"
                  "datasets given in the chunk_sizes list."

      }
    """,
            process_includes=True,
        )

        # The script usage
        usage = (
            "usage: dials.split_experiments [options] [param.phil] "
            "experiments1.expt experiments2.expt reflections1.refl "
            "reflections2.refl..."
        )

        # Create the parser
        self.parser = OptionParser(
            usage=usage,
            phil=phil_scope,
            read_reflections=True,
            read_experiments=True,
            check_format=False,
            epilog=help_message,
        )
Ejemplo n.º 60
0
def run(args):
    from dials.util.options import OptionParser
    from dials.util.options import flatten_experiments

    usage = "dials.background [options] image_*.cbf"

    parser = OptionParser(usage=usage,
                          phil=phil_scope,
                          read_experiments=True,
                          epilog=help_message)

    params, options = parser.parse_args(show_diff_phil=True)

    # Ensure we have either a data block or an experiment list
    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) != 1:
        parser.print_help()
        return

    imagesets = experiments.imagesets()

    if len(imagesets) != 1:
        raise Sorry(
            "Please pass an experiment list that contains a single imageset")
    imageset = imagesets[0]

    first, last = imageset.get_scan().get_image_range()
    images = range(first, last + 1)

    if params.images:
        if min(params.images) < first or max(params.images) > last:
            raise Sorry("image outside of scan range")
        images = params.images

    d_spacings = []
    intensities = []
    sigmas = []

    for indx in images:
        print("For image %d:" % indx)
        indx -= first  # indices passed to imageset.get_raw_data start from zero
        d, I, sig = background(imageset,
                               indx,
                               n_bins=params.n_bins,
                               mask_params=params.masking)

        print("%8s %8s %8s" % ("d", "I", "sig"))
        for j in range(len(I)):
            print("%8.3f %8.3f %8.3f" % (d[j], I[j], sig[j]))

        d_spacings.append(d)
        intensities.append(I)
        sigmas.append(sig)

    if params.plot:
        from matplotlib import pyplot

        fig = pyplot.figure()
        ax = fig.add_subplot(111)
        ax.set_xlabel(r"resolution ($\AA$)")
        ax.set_ylabel(r"$\langle I_b \rangle$")
        for d, I, sig in zip(d_spacings, intensities, sigmas):
            ds2 = 1 / flex.pow2(d)
            ax.plot(ds2, I)
        xticks = ax.get_xticks()

        x_tick_labs = [
            "" if e <= 0.0 else "{:.2f}".format(math.sqrt(1.0 / e))
            for e in xticks
        ]
        ax.set_xticklabels(x_tick_labs)

        pyplot.show()