Exemple #1
0
def run(args):

    # create parser
    #logger = multi_out() #logging.getLogger('main')
    #logger.register('stdout', sys.stdout)
    #logger = null_out()
    logger = multi_out()
    logger.register('stderr', sys.stderr)
    logger2 = multi_out()
    logger2.register('stdout', sys.stdout)
    #only omegalyze output is sent to stdout for backward compatibility with
    #  MolProbity website

    parser = CCTBXParser(program_class=omegalyze.Program, logger=logger)
    namespace = parser.parse_args(sys.argv[1:])

    # start program
    print('Starting job', file=logger)
    print('=' * 79, file=logger)
    task = omegalyze.Program(parser.data_manager,
                             parser.working_phil.extract(),
                             logger=logger2)

    # validate inputs
    task.validate()

    # run program
    task.run()

    # stop timer
    print('', file=logger)
    print('=' * 79, file=logger)
    print('Job complete', file=logger)
    show_total_time(out=logger)
Exemple #2
0
def run(args):
    "The main program, if run from CCTBX / PHENIX."
    logger = multi_out()
    logger.register('stderr', sys.stderr)
    logger2 = multi_out()
    logger2.register('stdout', sys.stdout)
    logger3 = multi_out()
    # TO DIAGNOSE OPTIONS TROUBLES:
    # logger3.register('verbiage', open("suitename.stderr.log", "w"))

    parser = dualparse.parseArgs(Program, logger3)
    working_phil = parser.working_phil
    options = working_phil.extract().suitename

    # now we call into the core of suitename itself
    if options.version:
        print(version, file=logger2)
        return
    if options.infile == "" or options.infile == "-" or options.residuein or options.suitein:
        # let the core figure out the input
        main(optionsIn=options, outFile=logger2, errorFile=logger)
    else:
        type, ext = analyzeFileType(options.infile)
        if type == "":
            logger.write("File extension " + str(ext) + " not recognized\n")
            return
        if type == "pdb":
            suites.main(options=options, outFile=logger2, errorFile=logger)
        else:
            # help the core figure out the input file type
            if type == "kinemage":
                options.suitein = True
            elif type == "dangle":
                options.residuein = True
            main(optionsIn=options, outFile=logger2, errorFile=logger)
Exemple #3
0
def run(args):

    # create parser
    logger = multi_out()
    logger.register('stderr', sys.stderr)
    logger2 = multi_out()
    logger2.register('stdout', sys.stdout)

    parser = CCTBXParser(program_class=cablam.Program, logger=logger)
    namespace = parser.parse_args(sys.argv[1:])

    # start program
    print('Starting job', file=logger)
    print('=' * 79, file=logger)
    task = cablam.Program(parser.data_manager,
                          parser.working_phil.extract(),
                          logger=logger2)

    # validate inputs
    task.validate()

    # run program
    task.run()

    # stop timer
    print('', file=logger)
    print('=' * 79, file=logger)
    print('Job complete', file=logger)
    show_total_time(out=logger)
Exemple #4
0
    def read_models(self):
        # Read Nat's reference model from an MTZ file.  XXX The observation
        # type is given as F, not I--should they be squared?  Check with Nat!
        log = open("%s.log" % self.params.output.prefix, "w")
        out = multi_out()
        out.register("log", log, atexit_send_to=None)
        out.register("stdout", sys.stdout)
        print("I model", file=out)
        if self.params.model is not None:
            from xfel.merging.general_fcalc import run as run_fmodel
            i_model = run_fmodel(self.params)
            self.params.target_unit_cell = i_model.unit_cell()
            self.params.target_space_group = i_model.space_group_info()
            i_model.show_summary()
        else:
            i_model = None

        print("Target unit cell and space group:", file=out)
        print("  ", self.params.target_unit_cell, file=out)
        print("  ", self.params.target_space_group, file=out)
        from xfel.command_line.cxi_merge import consistent_set_and_model
        self.miller_set, self.i_model = consistent_set_and_model(
            self.params, i_model)
        self.frame_files = get_observations(self.params)
        self.out = out
Exemple #5
0
    def start(self, interval=None, opts=None):
        self.stop()

        if self.opts.sdir!=opts.sdir or self.opts.ddir!=opts.ddir:
            self._transferred = {}
            self._bsslog_cache = {}
            self.skipped = set()

        self.opts = opts

        self.log_out = multi_out()
        self.log_out.register("stdout", sys.stdout)
        self.log_out.register("file", open(os.path.join(self.opts.sdir, logfile_name), "a"))

        self.log_out.write(time.strftime("# DATA SYNC GUI - Starting at %Y-%m-%d %H:%M:%S\n"))
        self.opts.show(self.log_out)
        self.log_out.write("\n")
        self.log_out.flush()

        self.keep_going = True
        self.stop_now = False # for emergency stop
        self.running = True
        if interval is not None:
            self.interval = interval

        self.thread = threading.Thread(None, self.run)
        self.thread.daemon = True
        self.thread.start()
Exemple #6
0
def get_input(args):
  if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
    print_help()
  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = libtbx.phil.command_line.argument_interpreter(
      master_phil=experimental_info.master_params,
      home_scope="experiment")

    print >> log, "#phil __OFF__"
    print >> log, "=========================="
    print >> log, "           SDP            "
    print >> log, "   SAXS Data Processing   "
    print >> log, "=========================="
    print >> log


    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = experimental_info.master_params.fetch(sources=phil_objects)
    params = effective_params.extract()
    print >> log, "#phil __ON__"
    new_params =  experimental_info.master_params.format(python_object=params)
    new_params.show(out=log,expert_level=1)
    print >> log, "#phil __END__"
    run( params, log )
Exemple #7
0
 def run(self):
     self.callback_start()
     self._stdout = multi_out()
     self._tmp_stdout = open(self.stdout_file, "w")
     self._stdout.register("Communication log", self._tmp_stdout)
     old_stdout = sys.stdout
     sys.stdout = stdout_redirect(self)
     import libtbx.callbacks
     libtbx.call_back.register_handler(self.callback_wrapper)
     try:
         return_value = self.target()
     except Abort:  # FIXME why is this not working properly?
         self.callback_abort()
     except Exception as e:
         print(type(e).__name__, file=sys.stderr)
         if (type(e).__name__ == "Abort"):
             self.callback_abort()
         else:
             if e.__class__.__module__ == "Boost.Python":
                 e = RuntimeError("Boost.Python.%s: %s" %
                                  (e.__class__.__name__, str(e)))
             elif hasattr(e, "reset_module"):
                 e.reset_module()
             traceback_str = "\n".join(
                 traceback.format_tb(sys.exc_info()[2]))
             self.callback_error(e, traceback_str)
     else:
         #time.sleep(1)
         self.callback_final(return_value)
     sys.stdout = old_stdout
Exemple #8
0
def test_reference_model(mon_lib_srv, ener_lib, prefix="tst_reference_model"):
    from mmtbx.geometry_restraints.torsion_restraints.tst_reference_model import \
        model_raw_records, reference_raw_records
    from mmtbx.geometry_restraints.torsion_restraints.reference_model import \
      reference_model
    import mmtbx.model
    # mstream = StringIO()
    from libtbx.utils import multi_out
    mstream = multi_out()
    mstream.register("stdout", sys.stdout)
    mstream_file_name = "polder.log"
    mstreamfile = open(mstream_file_name, "w")
    mstream.register("logfile", mstreamfile)

    work_params = mmtbx.model.manager.get_default_pdb_interpretation_params()
    work_params.reference_model.enabled = True
    work_params.reference_model.fix_outliers = False
    pdb_inp = iotbx.pdb.input(lines=model_raw_records.split('\n'),
                              source_info=None)
    model = mmtbx.model.manager(model_input=pdb_inp,
                                process_input=True,
                                pdb_interpretation_params=work_params)
    reference_hierarchy_list = []
    tmp_hierarchy = iotbx.pdb.input(
        source_info=None,
        lines=reference_raw_records.split('\n')).construct_hierarchy()
    reference_hierarchy_list.append(tmp_hierarchy)
    rm = reference_model(model=model,
                         reference_hierarchy_list=reference_hierarchy_list,
                         params=work_params.reference_model,
                         log=mstream)
    assert rm.get_n_proxies() == 5, "Got %d, expected 5" % rm.get_n_proxies()
    geometry, xrs = make_initial_grm(mon_lib_srv, ener_lib, model_raw_records)
    geometry.adopt_reference_dihedral_manager(rm)
    make_geo_pickle_unpickle(geometry, xrs, prefix)
Exemple #9
0
def DataManager(datatypes=None, phil=None, logger=None):
    '''
  Function for dynamically creating a DataManager instance that supports a
  specific set of datatypes.

  All DataManager modules in iotbx/data_manager follow this pattern:
    filename = <datatype>.py -> module name = iotbx.data_manager.<datatype>
    DataManagerBase subclass name = <Datatype>DataManager

  So for the models, the filename is iotbx/data_manager/model.py and in that
  file, there should be a subclass of DataManagerBase named ModelDataManager
  '''

    if logger is None:
        logger = multi_out()

    # set default if necessary
    if datatypes is None:
        datatypes = default_datatypes

    # get classes
    modules = load_datatype_modules(datatypes)
    manager_classes = []
    for datatype in datatypes:
        module_name = 'iotbx.data_manager.' + datatype
        class_name = datatype.capitalize() + 'DataManager'
        if '_' in datatype:  # real_map becomes RealMapDataManager
            class_name = ''.join(
                tmp_str.capitalize()
                for tmp_str in datatype.split('_')) + 'DataManager'
        manager_classes.append(getattr(sys.modules[module_name], class_name))

    # check inheritance and add datatypes if necessary
    class_datatypes = set()
    parent_classes = []
    for manager_class in manager_classes:
        if hasattr(manager_class, 'datatype'):
            class_datatypes.add(manager_class.datatype)
        # get full inheritance order and check
        for parent_class in inspect.getmro(manager_class)[1:]:
            if hasattr(parent_class, 'datatype'):
                class_datatypes.add(parent_class.datatype)
                parent_classes.append(parent_class)
                try:  # remove parent class and add back later
                    manager_classes.remove(parent_class)
                except ValueError:  # parent class already removed
                    pass
    datatypes = list(class_datatypes)

    # add mixin classes if necessary
    mixin_classes = []
    if 'real_map' in datatypes and 'map_coefficients' in datatypes:
        importlib.import_module('.common', package='iotbx.data_manager')
        mixin_classes.append(
            getattr(sys.modules['iotbx.data_manager.common'], 'map_mixins'))

    # construct new class and return instance
    classes = tuple(manager_classes + parent_classes + mixin_classes)
    data_manager_class = type('DataManager', classes, dict())
    return data_manager_class(datatypes=datatypes, phil=phil, logger=logger)
 def run (self) :
   self.callback_start()
   self._stdout = multi_out()
   self._tmp_stdout = open(self.stdout_file, "w")
   self._stdout.register("Communication log", self._tmp_stdout)
   old_stdout = sys.stdout
   sys.stdout = stdout_redirect(self)
   import libtbx.callbacks
   libtbx.call_back.register_handler(self.callback_wrapper)
   try :
     return_value = self.target()
   except Abort : # FIXME why is this not working properly?
     self.callback_abort()
   except Exception, e :
     print >> sys.stderr, type(e).__name__
     if (type(e).__name__ == "Abort") :
       self.callback_abort()
     else :
       if e.__class__.__module__ == "Boost.Python" :
         e = RuntimeError("Boost.Python.%s: %s" % (e.__class__.__name__,
           str(e)))
       elif hasattr(e, "reset_module") :
         e.reset_module()
       traceback_str = "\n".join(traceback.format_tb(sys.exc_info()[2]))
       self.callback_error(e, traceback_str)
Exemple #11
0
def run_program(program_class=None,
                custom_process_arguments=None,
                args=None,
                logger=None):
    '''
  Function for running programs using CCTBXParser and the program template

  :param program_class:  ProgramTemplate type (required)
  :param custom_process_arguments:
                         Custom function to parse unknown arguments (optional)
  :param args:           list of command-line arguments (optional)
  :param logger:         logger (e.g. multi_out) for output (optional)
  :rtype:                whatever is returned from program_class.get_results()
  '''

    assert program_class is not None

    if args is None:
        args = sys.argv[1:]

    # create logger
    if logger is None:
        logger = multi_out()
        logger.register('stdout', sys.stdout)
        logger.register('parser_log', StringIO())

    # start timer
    t = show_times(out=logger)

    # create parser
    parser = CCTBXParser(program_class=program_class,
                         custom_process_arguments=custom_process_arguments,
                         logger=logger)
    namespace = parser.parse_args(args)

    # start program
    print('Starting job', file=logger)
    print('=' * 79, file=logger)
    task = program_class(parser.data_manager,
                         parser.working_phil.extract(),
                         master_phil=parser.master_phil,
                         logger=logger)

    # validate inputs
    task.validate()

    # run program
    task.run()

    # clean up (optional)
    task.clean_up()

    # stop timer
    print('', file=logger)
    print('=' * 79, file=logger)
    print('Job complete', file=logger)
    t()

    return task.get_results()
    def initialize(self, args):
        import iotbx.phil
        from xfel.command_line.cxi_merge import master_phil
        if ("--help" in args):
            iotbx.phil.parse(master_phil).show(attributes_level=2)
            return
        phil = iotbx.phil.process_command_line(
            args=args, master_string=master_phil).show()
        work_params = phil.work.extract()
        from xfel.merging.phil_validation import application
        application(work_params)

        if ((work_params.d_min is None) or (work_params.data is None)
                or ((work_params.model is None)
                    and work_params.scaling.algorithm != "mark1")):
            command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
            raise Usage(command_name + " "
                        "d_min=4.0 "
                        "data=~/scratch/r0220/006/strong/ "
                        "model=3bz1_3bz2_core.pdb")
        if ((work_params.rescale_with_average_cell)
                and (not work_params.set_average_unit_cell)):
            raise Usage(
                "If rescale_with_average_cell=True, you must also specify " +
                "set_average_unit_cell=True.")
        if [
                work_params.raw_data.sdfac_auto,
                work_params.raw_data.sdfac_refine,
                work_params.raw_data.errors_from_sample_residuals
        ].count(True) > 1:
            raise Usage(
                "Specify only one of sdfac_auto, sdfac_refine or errors_from_sample_residuals."
            )

        # Read Nat's reference model from an MTZ file.  XXX The observation
        # type is given as F, not I--should they be squared?  Check with Nat!
        log = open("%s.log" % work_params.output.prefix, "w")
        out = multi_out()
        out.register("log", log, atexit_send_to=None)
        out.register("stdout", sys.stdout)
        print >> out, "I model"
        if work_params.model is not None:
            from xfel.merging.general_fcalc import run as run_fmodel
            i_model = run_fmodel(work_params)
            work_params.target_unit_cell = i_model.unit_cell()
            work_params.target_space_group = i_model.space_group_info()
            i_model.show_summary()
        else:
            i_model = None

        print >> out, "Target unit cell and space group:"
        print >> out, "  ", work_params.target_unit_cell
        print >> out, "  ", work_params.target_space_group
        from xfel.command_line.cxi_merge import consistent_set_and_model
        self.miller_set, self.i_model = consistent_set_and_model(
            work_params, i_model)
        self.work_params = work_params
        self.frame_files = get_observations(work_params)
        self.out = out
Exemple #13
0
def set_log(prefix, i_current, i_total):
  log = multi_out()
  fo = open("%s.tls.%s_of_%s"%(prefix,str(i_current),i_total),"w")
  log.register(
    label       = "log_buffer",
    file_object = fo)
  sys.stderr = log
  return log
Exemple #14
0
def run(args):
    # create parser
    logger = multi_out()  #logging.getLogger('main')
    logger.register('stdout', sys.stdout)

    parser = CCTBXParser(program_class=atom_selection.Program,
                         custom_process_arguments=custom_args_proc,
                         logger=logger)
    #
    # Stick in additional keyword args
    # They going to end up in namespace.cryst1_replacement_buffer_layer etc
    # file_name is obsolet, parser takes care of it putting it in data manager
    # inselections is going to be handled by custom_args_proc function
    # because it is intended to be positional argument
    #
    # !!! This is done here for legacy support and illustrative purposes.
    # Don't do it anywhere else, since we strive to have the same
    # command-line flags across all programs, like --overwrite etc.
    parser.add_argument(
        "--cryst1-replacement-buffer-layer",
        action="store",
        type=float,
        help="replace CRYST1 with pseudo unit cell covering the selected"
        " atoms plus a surrounding buffer layer",
        default=None)
    parser.add_argument("--write-pdb-file",
                        "--write_pdb_file",
                        "--write_pdb-file",
                        "--write-pdb_file",
                        action="store",
                        help="write selected atoms to new PDB file",
                        default=None)

    namespace = parser.parse_args(sys.argv[1:])

    # start program
    print('Starting job', file=logger)
    print('=' * 79, file=logger)

    task = atom_selection.Program(parser.data_manager,
                                  parser.working_phil.extract(),
                                  master_phil=parser.master_phil,
                                  logger=logger)

    # validate inputs
    task.validate()

    # run program
    task.run()

    # stop timer
    print('', file=logger)
    print('=' * 79, file=logger)
    print('Job complete', file=logger)
    show_total_time(out=logger)
 def __call__ (self) :
   if (self.log_file is not None) :
     log = open(self.log_file, "w")
     new_out = multi_out()
     new_out.register("log", log)
     new_out.register("stdout", sys.stdout)
     sys.stdout = new_out
     self._out = new_out
   result = self.run()
   easy_pickle.dump(self.file_name, result)
   if (self._out is not None) and (not getattr(self._out, "closed", False)) :
     self._out.flush()
     # FIXME
     #self._out.close()
   return result
def run(args):

    logger = multi_out()  #logging.getLogger('main')
    logger.register('stdout', sys.stdout)

    # replace command-line parser
    # ---------------------------------------------------------------------------
    if (len(args) == 0):
        show_usage(logger=logger)
        sys.exit()

    input_objects = iotbx.phil.process_command_line_with_files(
        args=args,
        master_phil=prepare_pdb_deposition.master_params,
        pdb_file_def='input.model_file',
        seq_file_def='input.sequence_file')

    # get program settings
    params = input_objects.work.extract()

    # get files (will be handled by task.validate)
    # already
    # if (params.input.model_file is None):
    #   raise Sorry('One model file is required.')
    # if (params.input.sequence_file is None):
    #   raise Sorry('One sequence file is required.')

    pdb_input = iotbx.pdb.input(params.input.model_file)
    model = mmtbx.model.manager(model_input=pdb_input, log=logger)

    sequence = any_file(params.input.sequence_file)
    sequence.check_file_type('seq')
    sequence = sequence.file_object

    # construct data manager
    data_manager = DataManager()
    data_manager.add_model(params.input.model_file, model)
    data_manager.add_sequence(params.input.sequence_file, sequence)

    # ---------------------------------------------------------------------------
    # start program
    task = prepare_pdb_deposition.Program(data_manager, params, logger=logger)

    # validate inputs
    task.validate()

    # run program
    task.run()
Exemple #17
0
 def __call__(self):
     if (self.log_file is not None):
         log = open(self.log_file, "w")
         new_out = multi_out()
         new_out.register("log", log)
         new_out.register("stdout", sys.stdout)
         sys.stdout = new_out
         self._out = new_out
     result = self.run()
     easy_pickle.dump(self.file_name, result)
     if (self._out
             is not None) and (not getattr(self._out, "closed", False)):
         self._out.flush()
         # FIXME
         #self._out.close()
     return result
    def __init__(self, data_manager, params, master_phil=None, logger=None):
        '''
    Common constructor for all programs

    This is supposed to be lightweight. Custom initialization, if necessary,
    should be handled by the custom_init function. Developers should not need to
    override this function.

    Parameters
    ----------
    data_manager :
      An instance of the DataManager (libtbx/data_manager.py) class containing
      data structures from file input
    params :
      An instance of PHIL
    logger :
      Standard Python logger (from logging module), optional. A logger will be
      created if it is not provided.

    '''

        self.data_manager = data_manager
        self.master_phil = master_phil
        self.params = params
        self.logger = logger

        if self.logger is None:
            self.logger = multi_out()

        # master_phil should be provided by CCTBXParser or GUI because of
        # potential PHIL extensions
        if self.master_phil is None:
            self.master_phil = libtbx.phil.parse(self.master_phil_str,
                                                 process_includes=True)

        # set DataManager defaults
        if self.data_manager is not None:
            self.data_manager.set_default_output_filename(
                self.get_default_output_filename())
            try:
                self.data_manager.set_overwrite(self.params.output.overwrite)
            except AttributeError:
                pass
            self.data_manager.set_program(self)

        # optional initialization
        self.custom_init()
Exemple #19
0
def run (args, out=None) :
  if (out is None) : out = sys.stdout
  usage_string = """\
mmtbx.prune_model model.pdb data.mtz [options...]

Filters protein residues based on CC to 2mFo-DFc map and absolute
(sigma-scaled) values in 2mFo-DFc and mFo-DFc maps.  For fast automatic
correction of MR solutions after initial refinement (ideally with rotamer
correction) to remove spurious loops and sidechains.
"""
  from mmtbx.command_line import load_model_and_data
  cmdline = load_model_and_data(
    args=args,
    master_phil=get_master_phil(),
    out=out,
    process_pdb_file=False,
    create_fmodel=True)
  params = cmdline.params
  fmodel = cmdline.fmodel
  if (params.output.file_name is None) :
    base_name = os.path.basename(params.input.pdb.file_name[0])
    params.output.file_name = os.path.splitext(base_name)[0] + "_pruned.pdb"
  log_file = os.path.splitext(os.path.basename(params.output.file_name))[0] + \
    ".log"
  log = open(log_file, "w")
  out2 = multi_out()
  out2.register("out", out)
  out2.register("log", log)
  map_helper = fmodel.electron_density_map()
  f_map_coeffs = map_helper.map_coefficients(map_type="2mFo-DFc")
  diff_map_coeffs = map_helper.map_coefficients(map_type="mFo-DFc")
  model_map_coeffs = map_helper.map_coefficients(map_type="Fc")
  result = prune_model(
    f_map_coeffs=f_map_coeffs,
    diff_map_coeffs=diff_map_coeffs,
    model_map_coeffs=model_map_coeffs,
    pdb_hierarchy=cmdline.pdb_hierarchy,
    params=params.prune).process_residues(out=out2)
  f = open(params.output.file_name, "w")
  f.write("REMARK edited by mmtbx.prune_model\n")
  f.write(cmdline.pdb_hierarchy.as_pdb_string(
    crystal_symmetry=fmodel.xray_structure))
  f.close()
  log.close()
  print >> out, "Wrote %s" % params.output.file_name
  return params.output.file_name
Exemple #20
0
def run(args, out=None):
    if (out is None): out = sys.stdout
    usage_string = """\
mmtbx.prune_model model.pdb data.mtz [options...]

Filters protein residues based on CC to 2mFo-DFc map and absolute
(sigma-scaled) values in 2mFo-DFc and mFo-DFc maps.  For fast automatic
correction of MR solutions after initial refinement (ideally with rotamer
correction) to remove spurious loops and sidechains.
"""
    from mmtbx.command_line import load_model_and_data
    cmdline = load_model_and_data(args=args,
                                  master_phil=get_master_phil(),
                                  out=out,
                                  process_pdb_file=False,
                                  create_fmodel=True)
    params = cmdline.params
    fmodel = cmdline.fmodel
    if (params.output.file_name is None):
        base_name = os.path.basename(params.input.pdb.file_name[0])
        params.output.file_name = os.path.splitext(
            base_name)[0] + "_pruned.pdb"
    log_file = os.path.splitext(os.path.basename(params.output.file_name))[0] + \
      ".log"
    log = open(log_file, "w")
    out2 = multi_out()
    out2.register("out", out)
    out2.register("log", log)
    map_helper = fmodel.electron_density_map()
    f_map_coeffs = map_helper.map_coefficients(map_type="2mFo-DFc")
    diff_map_coeffs = map_helper.map_coefficients(map_type="mFo-DFc")
    model_map_coeffs = map_helper.map_coefficients(map_type="Fc")
    result = prune_model(f_map_coeffs=f_map_coeffs,
                         diff_map_coeffs=diff_map_coeffs,
                         model_map_coeffs=model_map_coeffs,
                         pdb_hierarchy=cmdline.pdb_hierarchy,
                         params=params.prune).process_residues(out=out2)
    f = open(params.output.file_name, "w")
    f.write("REMARK edited by mmtbx.prune_model\n")
    f.write(
        cmdline.pdb_hierarchy.as_pdb_string(
            crystal_symmetry=fmodel.xray_structure))
    f.close()
    log.close()
    print >> out, "Wrote %s" % params.output.file_name
    return params.output.file_name
Exemple #21
0
def DataManager(datatypes=None, phil=None, logger=None):
    '''
  Function for dynamically creating a DataManager instance that supports a
  specific set of datatypes.

  All DataManager modules in iotbx/data_manager follow this pattern:
    filename = <datatype>.py -> module name = iotbx.data_manager.<datatype>
    DataManagerBase subclass name = <Datatype>DataManager

  So for the models, the filename is iotbx/data_manager/model.py and in that
  file, there should be a subclass of DataManagerBase named ModelDataManager
  '''

    if (logger is None):
        logger = multi_out()

    # set default if necessary
    if (datatypes is None):
        datatypes = default_datatypes

    # get classes
    modules = load_datatype_modules(datatypes)
    manager_classes = list()
    for datatype in datatypes:
        module_name = 'iotbx.data_manager.' + datatype
        class_name = datatype.capitalize() + 'DataManager'
        if ('_' in datatype):  # real_map becomes RealMapDataManager
            class_name = ''.join(
                tmp_str.capitalize()
                for tmp_str in datatype.split('_')) + 'DataManager'
        manager_classes.append(getattr(sys.modules[module_name], class_name))

    # check inheritance and add datatypes if necessary
    class_datatypes = set()
    for manager_class in manager_classes:
        if (hasattr(manager_class, 'datatype')):
            class_datatypes.add(manager_class.datatype)
        # get full inheritance order and check
        for parent_class in inspect.getmro(manager_class)[1:]:
            if (hasattr(parent_class, 'datatype')):
                class_datatypes.add(parent_class.datatype)
    datatypes = list(class_datatypes)

    # construct new class and return instance
    data_manager_class = type('DataManager', tuple(manager_classes), dict())
    return data_manager_class(datatypes=datatypes, phil=phil, logger=logger)
def run(params):
    log_out = multi_out()
    log_out.register("log", open(params.logfile, "w"), atexit_send_to=None)
    log_out.register("stdout", sys.stdout)

    libtbx.phil.parse(master_params_str).format(params).show(out=log_out, prefix=" ")

    xac_files = read_path_list(params.lstin, only_exists=True, err_out=log_out)

    if len(xac_files) == 0:
        print >>log_out, "No (existing) files in the list: %s" % params.lstin
        return
    
    if params.method == "brehm_diederichs":
        rb = BrehmDiederichs(xac_files, max_delta=params.max_delta,
                             d_min=params.d_min, min_ios=params.min_ios,
                             nproc=params.nproc, log_out=log_out)
    elif params.method == "selective_breeding":
        rb = KabschSelectiveBreeding(xac_files, max_delta=params.max_delta,
                                     d_min=params.d_min, min_ios=params.min_ios,
                                     nproc=params.nproc, log_out=log_out)
    elif params.method == "reference":
        rb = ReferenceBased(xac_files, params.reference_file, max_delta=params.max_delta,
                            d_min=params.d_min, min_ios=params.min_ios,
                            nproc=params.nproc, log_out=log_out)
    else:
        raise "Unknown method: %s" % params.method

    rb.assign_operators()
    new_files = rb.modify_xds_ascii_files()

    lstout = os.path.splitext(os.path.basename(params.lstin))[0]+"_reindexed.lst"
    ofs = open(lstout, "w")
    ofs.write("\n".join(new_files)+"\n")
    ofs.close()
    print >>log_out, "Reindexing done. For merging, use %s instead!" % lstout

    if params.method == "brehm_diederichs":
        print >>log_out, """
CCTBX-implementation (by Richard Gildea) of the "algorithm 2" of the following paper was used.
For publication, please cite:
 Brehm, W. and Diederichs, K. Breaking the indexing ambiguity in serial crystallography.
 Acta Cryst. (2014). D70, 101-109
 http://dx.doi.org/10.1107/S1399004713025431"""
    elif params.method == "selective_breeding":
        print >>log_out, """
Exemple #23
0
def test_phil():
    master_phil = libtbx.phil.parse(TestProgram.master_phil)
    required_output_phil = libtbx.phil.parse(ProgramTemplate.output_phil_str)
    master_phil.adopt_scope(required_output_phil)

    params = master_phil.fetch(working_phil).extract()
    logger = multi_out()
    logger.register('stdout', sys.stdout)
    test_program = TestProgram(None, params, master_phil, logger)

    full_phil = libtbx.phil.parse(test_program.get_program_phil_str())
    full = master_phil.fetch(full_phil).extract()
    assert full.parameter_a == 'not None'
    assert full.parameter_b == 5
    assert full.parameter_c is None
    assert 'parameter_c' in test_program.get_program_phil_str()
    assert 'parameter_c' not in test_program.get_program_phil_str(True)

    assert test_program.get_default_output_filename() == 'cctbx_program_000'
    assert test_program.get_default_output_filename(prefix='abc') == 'abc_000'
    assert test_program.get_default_output_filename(
        suffix='abc') == 'cctbx_programabc_000'
    assert test_program.get_default_output_filename(
        serial=999) == 'cctbx_program_999'
    assert test_program.get_default_output_filename(prefix='abc',
                                                    suffix='def',
                                                    serial=123) == 'abcdef_123'

    test_program.params.output.prefix = 'prefix'
    test_program.params.output.suffix = 'suffix'
    test_program.params.output.serial = 7
    assert test_program.get_default_output_filename() == 'prefixsuffix_007'
    assert test_program.get_default_output_filename(
        prefix='abc') == 'abcsuffix_007'
    assert test_program.get_default_output_filename(
        suffix='abc') == 'prefixabc_007'
    assert test_program.get_default_output_filename(
        serial=999) == 'prefixsuffix_999'

    try:
        test_program.get_default_output_filename(serial='abc')
    except ValueError as e:
        if str(e) != 'The serial argument should be an integer.':
            raise
def run (args, out=None) :
  if (out is None) : out = sys.stdout
  from mmtbx.building.alternate_conformations import single_residue
  import mmtbx.command_line
  cmdline = mmtbx.command_line.load_model_and_data(
    args=args,
    master_phil=master_phil(),
    process_pdb_file=True,
    create_fmodel=True,
    out=out,
    usage_string="""\
mmtbx.build_alt_confs_simple model.pdb data.mtz [options]

Simple tool for building alternate conformations by real-space refinement into
difference density.  Not intended for production use - use the program
mmtbx.build_alternate_conformations if you want refinement and post-processing.
""")
  params = cmdline.params
  validate_params(params)
  log = multi_out()
  log.register("stdout", out)
  log_file_name = os.path.splitext(params.output.file_name)[0] + ".log"
  logfile = open(log_file_name, "w")
  log.register("logfile", logfile)
  pdb_hierarchy, n_alternates = single_residue.build_cycle(
    pdb_hierarchy = cmdline.pdb_hierarchy,
    fmodel = cmdline.fmodel,
    geometry_restraints_manager = cmdline.geometry,
    params = params,
    cif_objects=cmdline.cif_objects,
    selection=params.selection,
    nproc=params.nproc,
    verbose=params.output.verbose,
    debug=params.output.debug,
    out=log)
  # TODO real-space refinement of multi-conformer model
  f = open(params.output.file_name, "w")
  f.write(pdb_hierarchy.as_pdb_string(
    crystal_symmetry=cmdline.fmodel.xray_structure))
  f.close()
  make_header("Building complete", out=out)
  print >> log, ""
  print >> log, "Wrote %s" % params.output.file_name
  print >> log, "You MUST refine this model before using it!"
def run(args, out=None):
    if (out is None): out = sys.stdout
    from mmtbx.building.alternate_conformations import single_residue
    import mmtbx.command_line
    cmdline = mmtbx.command_line.load_model_and_data(args=args,
                                                     master_phil=master_phil(),
                                                     process_pdb_file=True,
                                                     create_fmodel=True,
                                                     out=out,
                                                     usage_string="""\
mmtbx.build_alt_confs_simple model.pdb data.mtz [options]

Simple tool for building alternate conformations by real-space refinement into
difference density.  Not intended for production use - use the program
mmtbx.build_alternate_conformations if you want refinement and post-processing.
""")
    params = cmdline.params
    validate_params(params)
    log = multi_out()
    log.register("stdout", out)
    log_file_name = os.path.splitext(params.output.file_name)[0] + ".log"
    logfile = open(log_file_name, "w")
    log.register("logfile", logfile)
    pdb_hierarchy, n_alternates = single_residue.build_cycle(
        pdb_hierarchy=cmdline.pdb_hierarchy,
        fmodel=cmdline.fmodel,
        geometry_restraints_manager=cmdline.geometry,
        params=params,
        cif_objects=cmdline.cif_objects,
        selection=params.selection,
        nproc=params.nproc,
        verbose=params.output.verbose,
        debug=params.output.debug,
        out=log)
    # TODO real-space refinement of multi-conformer model
    f = open(params.output.file_name, "w")
    f.write(
        pdb_hierarchy.as_pdb_string(
            crystal_symmetry=cmdline.fmodel.xray_structure))
    f.close()
    make_header("Building complete", out=out)
    print >> log, ""
    print >> log, "Wrote %s" % params.output.file_name
    print >> log, "You MUST refine this model before using it!"
def run(args):
  # processing command-line stuff, out of the object
  log = multi_out()
  log.register("stdout", sys.stdout)
  if len(args) == 0:
    format_usage_message(log)
    return
  inputs = mmtbx.utils.process_command_line_args(args=args,
      master_params=master_params())
  work_params = inputs.params.extract()
  inputs.params.show(prefix=" ", out=log)
  pdb_file_names = list(inputs.pdb_file_names)
  if work_params.file_name is not None:
    pdb_file_names += work_params.file_name
  if len(pdb_file_names) == 0:
    raise Sorry("No PDB file specified")
  if work_params.output_prefix is None:
    work_params.output_prefix = os.path.basename(pdb_file_names[0])
  log_file_name = "%s.log" % work_params.output_prefix
  logfile = open(log_file_name, "w")
  log.register("logfile", logfile)
  if work_params.loop_idealization.output_prefix is None:
    work_params.loop_idealization.output_prefix = "%s_rama_fixed" % work_params.output_prefix
  pdb_combined = iotbx.pdb.combine_unique_pdb_files(file_names=pdb_file_names)
  pdb_input = iotbx.pdb.input(source_info=None,
    lines=flex.std_string(pdb_combined.raw_records))
  mi_object = model_idealization(
      pdb_input=pdb_input,
      cif_objects=inputs.cif_objects,
      params=work_params,
      log=log,
      verbose=True)
  mi_object.run()
  mi_object.print_stat_comparison()
  print >> log, "RMSD from starting model (backbone, all): %.4f, %.4f" % (
      mi_object.get_rmsd_from_start(), mi_object.get_rmsd_from_start2())
  mi_object.print_runtime()
  # add hydrogens if needed ?
  print >> log, "All done."
  log.close()
Exemple #27
0
def test_phil():
    master_phil = libtbx.phil.parse(TestProgram.master_phil)
    required_output_phil = libtbx.phil.parse(ProgramTemplate.output_phil_str)
    master_phil.adopt_scope(required_output_phil)

    params = master_phil.fetch(working_phil).extract()
    logger = multi_out()
    logger.register('stdout', sys.stdout)
    test_program = TestProgram(None, params, master_phil, logger)

    full_phil = libtbx.phil.parse(test_program.get_program_phil_str())
    full = master_phil.fetch(full_phil).extract()
    assert (full.parameter_a == 'not None')
    assert (full.parameter_b == 5)
    assert (full.parameter_c is None)
    assert ('parameter_c' in test_program.get_program_phil_str())
    assert ('parameter_c' not in test_program.get_program_phil_str(True))

    assert (test_program.get_default_output_filename() == 'cctbx_program_000')
    test_program.params.output.prefix = 'prefix'
    test_program.params.output.suffix = 'suffix'
    test_program.params.output.serial = 7
    assert (test_program.get_default_output_filename() == 'prefixsuffix_007')
def run(args, out=sys.stdout):
  log = multi_out()
  log.register("stdout", out)
  log_file_name = "hydrogen_parameterization.log"
  logfile = open(log_file_name, "w")
  log.register("logfile", logfile)
  if (len(args) == 0):
    print >>log, legend
    return

  print >> log, "phenix.hydrogen_parameterization is running..."
  print >> log, "input parameters:\n", args

# parse through params --> switch off CDL or not
  params_line = grand_master_phil_str
  params = iotbx.phil.parse(
      input_string=params_line, process_includes=True).extract()
  params.pdb_interpretation.restraints_library.cdl=False

  processed_args = mmtbx.utils.process_command_line_args(
    args=args, log=null_out())
  pdb_filename = processed_args.pdb_file_names[0]
  mon_lib_srv = monomer_library.server.server()
  ener_lib = monomer_library.server.ener_lib()
  if pdb_filename is not None:
    processed_pdb_file = monomer_library.pdb_interpretation.process(
      params         = params.pdb_interpretation,
      mon_lib_srv    = mon_lib_srv,
      ener_lib       = ener_lib,
      file_name      = pdb_filename,
      force_symmetry = True)

  pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
  xray_structure = processed_pdb_file.xray_structure()
  if pdb_filename is not None:
    pdb_str = pdb_hierarchy.as_pdb_string()

  geometry_restraints = processed_pdb_file.geometry_restraints_manager(
    show_energies = False)
  restraints_manager = mmtbx.restraints.manager(
    geometry      = geometry_restraints,
    normalization = False)

  bond_proxies_simple, asu = restraints_manager.geometry.get_all_bond_proxies(
    sites_cart = xray_structure.sites_cart())
  angle_proxies = restraints_manager.geometry.get_all_angle_proxies()
  hd_selection = xray_structure.hd_selection()
  names = list(pdb_hierarchy.atoms().extract_name())
  sites_cart = xray_structure.sites_cart()
  #scatterers = xray_structure.scatterers()

  atoms_list = list(pdb_hierarchy.atoms_with_labels())

  print >>log, '\nNow determining connectivity table for H atoms...'
  connectivity = hydrogen_connectivity.determine_H_neighbors(
    geometry_restraints   = geometry_restraints,
    bond_proxies          = bond_proxies_simple,
    angle_proxies         = angle_proxies,
    hd_selection          = hd_selection,
    sites_cart            = sites_cart)

  print >>log, '\nNow determining the parameterization for H atoms...'
  h_parameterization = get_h_parameterization(
    connectivity   = connectivity,
    sites_cart     = sites_cart,
    idealize       = False)

  print >>log, '\nNow reconstructing H atoms...'
  long_distance_list = []
  unk_list = []
  for ih in h_parameterization.keys():
    residue = atoms_list[ih].resseq
    hp = h_parameterization[ih]
    h_obj = generate_H_positions(
      sites_cart        = sites_cart,
      ih                = ih,
      para_info         = hp)
    if(h_obj.distance is not None):
      print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
        residue, 'distance:', h_obj.distance
      if(h_obj.distance > 0.05):
        long_distance_list.append(ih)
    else:
      print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', residue
      unk_list.append(ih)

  # some informative output for residues with unknown algorithm
  print >>log, '*'*79
  print >>log, 'Warning: The following atoms where not assigned an H type'
  for ih in unk_list:
    residue = atoms_list[ih].resseq
    hp = h_parameterization[ih]
    print >> log, 'atom:', names[ih], 'residue:', residue, \
      'chain', atoms_list[ih].chain_id
  print >>log, '*'*79

  # some informative output for residues where position is NOT reproduced
  # -> wronlgy assigned
  print >>log, 'Warning: The position of the following H atoms was not reproduced'
  for ih in long_distance_list:
    residue = atoms_list[ih].resseq
    hp = h_parameterization[ih]
    h_obj = generate_H_positions(
      sites_cart        = sites_cart,
      ih                = ih,
      para_info         = hp)
    if(h_obj.distance is not None and h_obj.distance > 0.05):
      print >> log, hp.htype, 'atom:', names[ih]+' ('+str(ih)+ ') residue:', \
        residue, 'chain', atoms_list[ih].chain_id, 'distance:', h_obj.distance
  print >>log, '*'*79

  for ih in h_parameterization.keys():
    hp = h_parameterization[ih]
    print 'htype = ', hp.htype, 'a0 = ', hp.a0, 'a1 = ', hp.a1, 'a2 = ', hp.a2, \
      'a = ', hp.a, 'b = ', hp.b, 'h = ', hp.h, 'chi = ', hp.chi, 'eps = ', hp.eps, \
      'alpha = ', hp.alpha, 'dist_h =', hp.dist_h
Exemple #29
0
def run(args):

  if len(args)==0:
    master_params.show(expert_level=100)
  elif ( "--help" in args ):
    print "no help available"
  elif ( "--h" in args ):
    print "no help available"
  elif ( "--show_defaults" in args ):
    master_params.show(expert_level=0)
  elif ( "--show_defaults_all" in args ):
    master_params.show(expert_level=10)

  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    log_plots = StringIO()
    print >> log,"#phil __OFF__"
    print >> log
    print >> log, date_and_time()
    print >> log
    print >> log

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="scaling")

    reflection_file = None

    for arg in args:
      command_line_params = None
      arg_is_processed = False
      if arg == '--quiet':
        arg_is_processed = True
        ## The associated action with this keyword is implemented above
      if (os.path.isfile(arg)): ## is this a file name?
        ## Check if this is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
        except KeyboardInterrupt: raise
        except Exception : pass
        if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        ## Check if this file is a reflection file
        if command_line_params is None:
          reflection_file = reflection_file_reader.any_reflection_file(
            file_name=arg, ensure_read_access=False)
        if (reflection_file is not None):
          reflection_file = arg
          arg_is_processed = True
      ## If it is not a file, it must be a phil command
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown phil-file or phil-command:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file format or phil command: %s" % arg)


    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()


    ## Now please read in the reflections files

    ## get symmetry and cell data first please
    ## By default, the native cell and symmetry are used
    ## as reference
    crystal_symmetry_nat = None
    print params.scaling.input.xray_data.wavelength1.file_name
    crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
      file_name=params.scaling.input.xray_data.wavelength1.file_name)

    if params.scaling.input.xray_data.space_group is None:
      params.scaling.input.xray_data.space_group =\
        crystal_symmetry_nat.space_group_info()
      print >> log, "Using symmetry of native data"

    if params.scaling.input.xray_data.unit_cell is None:
      params.scaling.input.xray_data.unit_cell =\
        crystal_symmetry_nat.unit_cell()
      print >> log, "Using cell of native data"

    ## Check if a unit cell is defined
    if params.scaling.input.xray_data.space_group is None:
      raise Sorry("No space group defined")
    if params.scaling.input.xray_data.unit_cell is None:
      raise Sorry("No unit cell defined")


    crystal_symmetry = crystal_symmetry = crystal.symmetry(
      unit_cell =  params.scaling.input.xray_data.unit_cell,
      space_group_symbol = str(
        params.scaling.input.xray_data.space_group) )


    effective_params = master_params.fetch(sources=phil_objects)
    new_params = master_params.format(python_object=params)
    print >> log, "Effective parameters"
    print >> log, "#phil __ON__"
    new_params.show(out=log,expert_level=params.scaling.input.expert_level)
    print >> log, "#phil __END__"
    print >> log

    ## define a xray data server
    xray_data_server =  reflection_file_utils.reflection_file_server(
      crystal_symmetry = crystal_symmetry,
      force_symmetry = True,
      reflection_files=[])

    ## Read in native data and make appropriate selections
    miller_array_w1 = None
    miller_array_w1 = xray_data_server.get_xray_data(
      file_name = params.scaling.input.xray_data.wavelength1.file_name,
      labels = params.scaling.input.xray_data.wavelength1.labels,
      ignore_all_zeros = True,
      parameter_scope = 'scaling.input.SIR_scale.xray_data.native'
    )
    info_native = miller_array_w1.info()
    miller_array_w1=miller_array_w1.map_to_asu().select(
      miller_array_w1.indices()!=(0,0,0) )
    miller_array_w1 = miller_array_w1.select(
      miller_array_w1.data() > 0 )
    ## Convert to amplitudes
    if (miller_array_w1.is_xray_intensity_array()):
      miller_array_w1 = miller_array_w1.f_sq_as_f()
    elif (miller_array_w1.is_complex_array()):
      miller_array_w1 = abs(miller_array_w1)
    if not miller_array_w1.is_real_array():
      raise Sorry("miller_array_native is not a real array")
    miller_array_w1.set_info(info = info_native)



    ## Read in derivative data and make appropriate selections
    miller_array_w2 = None
    miller_array_w2 = xray_data_server.get_xray_data(
      file_name = params.scaling.input.xray_data.wavelength2.file_name,
      labels = params.scaling.input.xray_data.wavelength2.labels,
      ignore_all_zeros = True,
      parameter_scope = 'scaling.input.SIR_scale.xray_data.derivative'
    )
    info_w2 = miller_array_w2.info()
    miller_array_w2=miller_array_w2.map_to_asu().select(
      miller_array_w2.indices()!=(0,0,0) )
    miller_array_w2 = miller_array_w2.select(
      miller_array_w2.data() > 0 )
    ## Convert to amplitudes
    if (miller_array_w2.is_xray_intensity_array()):
      miller_array_w2 = miller_array_w2.f_sq_as_f()
    elif (miller_array_w2.is_complex_array()):
      miller_array_w2 = abs(miller_array_w2)
    if not miller_array_w2.is_real_array():
      raise Sorry("miller_array_derivative is not a real array")
    miller_array_w2.set_info(info = info_w2)

    ## Make sure we have anomalous diffs in both files
    assert miller_array_w1.anomalous_flag()
    assert miller_array_w2.anomalous_flag()


    ## Print info
    print >> log
    print >> log, "Wavelength 1"
    print >> log, "============"
    miller_array_w1.show_comprehensive_summary(f=log)
    print >> log
    w1_pre_scale = pre_scale.pre_scaler(
      miller_array_w1,
      params.scaling.input.scaling_strategy.pre_scaler_protocol,
      params.scaling.input.basic)
    miller_array_w1 =  w1_pre_scale.x1.deep_copy()
    del w1_pre_scale

    print >> log
    print >> log, "Wavelength 2"
    print >> log, "============"
    miller_array_w2.show_comprehensive_summary(f=log)
    print >> log
    w2_pre_scale = pre_scale.pre_scaler(
      miller_array_w2,
      params.scaling.input.scaling_strategy.pre_scaler_protocol,
      params.scaling.input.basic)
    miller_array_w2 =  w2_pre_scale.x1.deep_copy()
    del w2_pre_scale


    print >> log
    print >> log, "Checking for possible reindexing schemes"
    print >> log, "----------------------------------------"
    print >> log
    print >> log, "Reindexing operator derived as described in:"
    print >> log, "Grosse-Kunstleve, Afonine, Sauter & Adams. (2005)."
    print >> log, "  IUCr Computing Commission Newsletter 5."
    print >> log

    reindex_object = pair_analyses.reindexing(
       set_a=miller_array_w1,
       set_b=miller_array_w2,
       out=log)
    miller_array_w2 = reindex_object.select_and_transform()
    miller_array_w2.map_to_asu()

    print >> log
    print >> log, "Relative scaling of 2-wavelength mad data"
    print >> log, "-----------------------------------------"
    print >> log
    scaler = fa_estimation.combined_scaling(
      miller_array_w1,
      miller_array_w2,
      params.scaling.input.scaling_strategy.iso_protocol)

    miller_array_w1 = scaler.x1.deep_copy()
    miller_array_w2 = scaler.x2.deep_copy()

    del scaler

    print >> log
    print >> log, "Estimating f\" and f' ratios"
    print >> log, "----------------------------"
    print >> log



    # now things are scaled see if we can guestimate the ratio
    fdpratio = pair_analyses.f_double_prime_ratio(
      miller_array_w1,
      miller_array_w2)

    fpfdpratio = pair_analyses.delta_f_prime_f_double_prime_ratio(
      miller_array_w1,
      miller_array_w2)

    k1 = fdpratio.ratio
    k2 = fpfdpratio.ratio

    if k1 is not None:
      print >> log
      print >> log, "  The estimate of f\"(w1)/f\"(w2) is %3.2f"\
            %(fdpratio.ratio)
    if k2 is not None:
      print >> log, "  The estimate of (f'(w1)-f'(w2))/f\"(w2) is %3.2f"\
            %(fpfdpratio.ratio)
      print >> log
      print >> log, "  The quality of these estimates depends to a large extend"
      print >> log, "  on the quality of the data. If user supplied values"
      print >> log, "  of f\" and f' are given, they will be used instead "
      print >> log, "  of the estimates."
      print >> log

    if params.scaling.input.xray_data.wavelength1.f_double_prime is not None:
      if params.scaling.input.xray_data.wavelength2.f_double_prime is not None:
        k1 = (params.scaling.input.xray_data.wavelength1.f_double_prime/
              params.scaling.input.xray_data.wavelength2.f_double_prime)
        print >> log, "    Using user specified f\" values"
        print >> log, "      user specified f\"(w1)/f\"(w2) is %3.2f"\
              %(k1)
        print >> log
    if params.scaling.input.xray_data.wavelength1.f_prime is not None:
      if params.scaling.input.xray_data.wavelength2.f_prime is not None:
        if params.scaling.input.xray_data.wavelength2.f_double_prime is not None:

          k2 = (params.scaling.input.xray_data.wavelength1.f_prime-
                params.scaling.input.xray_data.wavelength2.f_prime)\
                /params.scaling.input.xray_data.wavelength2.f_double_prime
          print >> log, "    Using user specified f\" and f' values"
          print >> log, "     user specified f\"(w1)/f\"(w2) is %3.2f"\
                %(k2)
          print >> log



    fa_gen = fa_estimation.twmad_fa_driver(miller_array_w1,
                                           miller_array_w2,
                                           k1,
                                           k2,
                                           params.scaling.input.fa_estimation)

    print >> log
    print >> log, "writing mtz file"
    print >> log, "----------------"
    print >> log

    ## Please write out the abs_delta_f array

    fa =  fa_gen.fa_values

    mtz_dataset = fa.as_mtz_dataset(
      column_root_label='F'+params.scaling.input.output.outlabel)

    mtz_dataset.mtz_object().write(
      file_name=params.scaling.input.output.hklout)
def run(args, command_name="phenix.twin_map_utils"):
  log=sys.stdout
  params=None
  if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args):
    print_help(command_name=command_name)
  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="map_coefs")

    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()
    """
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    """
    # now get the unit cell from the pdb file

    hkl_xs = crystal_symmetry_from_any.extract_from(
      file_name=params.twin_utils.input.xray_data.file_name)
    pdb_xs = crystal_symmetry_from_any.extract_from(
      file_name=params.twin_utils.input.model.file_name)

    phil_xs=None
    if ([params.twin_utils.input.unit_cell,
         params.twin_utils.input.space_group]).count(None)<2:
      phil_xs = crystal.symmetry(
        unit_cell=params.twin_utils.input.unit_cell,
        space_group_info=params.twin_utils.input.space_group  )


    combined_xs = crystal.select_crystal_symmetry(
      None,phil_xs, [pdb_xs],[hkl_xs])

    # inject the unit cell and symmetry in the phil scope please
    params.twin_utils.input.unit_cell = combined_xs.unit_cell()
    params.twin_utils.input.space_group = \
      sgtbx.space_group_info( group = combined_xs.space_group() )

    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)

    if params.twin_utils.input.unit_cell is None:
      raise Sorry("unit cell not specified")
    if params.twin_utils.input.space_group is None:
      raise Sorry("space group not specified")
    if params.twin_utils.input.xray_data.file_name is None:
      raise Sorry("Xray data not specified")
    if params.twin_utils.input.model.file_name is None:
      raise Sorry("pdb file with  model not specified")

    #-----------------------------------------------------------
    #
    # step 1: read in the reflection file
    #
    phil_xs = crystal.symmetry(
      unit_cell=params.twin_utils.input.unit_cell,
      space_group_info=params.twin_utils.input.space_group  )

    xray_data_server =  reflection_file_utils.reflection_file_server(
      crystal_symmetry = phil_xs,
      force_symmetry = True,
      reflection_files=[])

    miller_array = None
    free_flags = None

    tmp_params = utils.data_and_flags_master_params().extract()
    # insert proper values please
    tmp_params.file_name = params.twin_utils.input.xray_data.file_name
    tmp_params.labels = params.twin_utils.input.xray_data.obs_labels
    tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name
    tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag

    tmp_object = utils.determine_data_and_flags( reflection_file_server = xray_data_server,
                                                 parameters = tmp_params, log=log )

    miller_array = tmp_object.extract_data()
    if miller_array.is_xray_intensity_array():
      miller_array = miller_array.f_sq_as_f()
    assert miller_array.is_xray_amplitude_array()

    free_flags = tmp_object.extract_flags(data = miller_array)
    print >> log
    print >> log, "Attempting to extract Free R flags"

    free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) )
    if free_flags is None:
      free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True)

    assert miller_array.observation_type() is not None

    print >> log
    print >> log, "Summary info of observed data"
    print >> log, "============================="
    miller_array.show_summary(f=log)
    print >> log

    if miller_array.indices().size() == 0:
      raise Sorry("No data available")

    #----------------------------------------------------------------
    # Step 2: get an xray structure from the PDB file
    #
    model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple(
      crystal_symmetry=phil_xs)
    print >> log, "Atomic model summary"
    print >> log, "===================="
    model.show_summary(f=log)
    print >> log


    #----------------------------------------------------------------
    # step 3: get the twin laws for this xs
    twin_laws = twin_analyses.twin_laws(
      miller_array,
      lattice_symmetry_max_delta=\
         params.twin_utils.parameters.twinning.max_delta,
      out=log)

    print >> log
    print >> log, "Preliminary data analyses"
    print >> log, "=========================="
    twin_laws.show(out=log)


    #---------
    # step 3:
    # make twin model managers for all twin laws
    print >> log
    print >> log, "Overall and bulk solvent scale paranmeters and twin fraction estimation"
    print >> log, "======================================================================="
    twin_models = []
    operator_count = 0

    if params.twin_utils.parameters.twinning.twin_law is not None:
      tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law )
      tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None)
      twin_laws.operators = [ tmp_law ]
    for twin_law in twin_laws.operators:
      operator_count += 1
      operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl()
      twin_model = twin_f_model.twin_model_manager(
        f_obs=miller_array,
        r_free_flags = free_flags,
        xray_structure=model,
        twin_law = twin_law.operator,
        detwin_mode = params.twin_utils.parameters.twinning.detwin_mode,
        out=log)


      print >> log, "--- bulk solvent scaling ---"
      twin_model.update_solvent_and_scale(update_f_part1=False)
      twin_model.r_values()
      twin_model.target()
      twin_model.show_k_sol_b_sol_b_cart_target()
      twin_model.show_essential()

      wfofc  = twin_model.map_coefficients(map_type="mFo-DFc"  )
      wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" )
      grad   = twin_model.map_coefficients(map_type="gradient"       )

      mtz_dataset = wtfofc.as_mtz_dataset(
        column_root_label="FWT")
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = wfofc,
        column_root_label = "DFWT"
        )
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = grad,
        column_root_label = "GRAD"
        )
      name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz"
      print >> log
      print >> log, "Writing %s for twin law %s"%(name,operator_hkl)
      print >> log
      mtz_dataset.mtz_object().write(
        file_name=name)

      if params.twin_utils.output.obs_and_calc is not None:
        # i want also a Fobs and Fmodel combined dataset please
        mtz_dataset = miller_array.as_mtz_dataset(
          column_root_label="FOBS")
        mtz_dataset = mtz_dataset.add_miller_array(
          miller_array = twin_model.f_model(),
          column_root_label="FMODEL")
        name = params.twin_utils.output.obs_and_calc
        mtz_dataset.mtz_object().write(
          file_name=name)

    if len(twin_laws.operators)==0:
      print >> log
      print >> log, "No twin laws were found"
      print >> log, "Performing maximum likelihood based bulk solvent scaling"
      f_model_object = f_model.manager(
        f_obs = miller_array,
        r_free_flags = free_flags,
        xray_structure = model )
      f_model_object.update_solvent_and_scale(out=log)
      tfofc =  f_model_object.map_coefficients(map_type="2mFobs-DFmodel")
      fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel")
      mtz_dataset = tfofc.as_mtz_dataset(
        column_root_label="FWT")
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = fofc,
        column_root_label = "DELFWT"
      )
      name = params.twin_utils.output.map_coeffs_root+"_ML.mtz"
      mtz_dataset.mtz_object().write(
        file_name=name)

      if params.twin_utils.output.obs_and_calc is not None:
        # i want also a Fobs and Fmodel combined dataset please
        mtz_dataset = miller_array.as_mtz_dataset(
          column_root_label="FOBS")
        mtz_dataset = mtz_dataset.add_miller_array(
          miller_array = f_model_object.f_model(),
          column_root_label="FMODEL")
        name = params.twin_utils.output.obs_and_calc
        mtz_dataset.mtz_object().write(
          file_name=name)

    print >> log
    print >> log
    print >> log, "All done \n"
    logfile = open(params.twin_utils.output.logfile,'w')
    print >> logfile,  string_buffer.getvalue()
    print >> log
Exemple #31
0
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  from xfel.merging.phil_validation import application
  application(work_params)
  if ("--help" in args) :
    libtbx.phil.parse(master_phil.show())
    return

  if ((work_params.d_min is None) or
      (work_params.data is None) or
      ( (work_params.model is None) and work_params.scaling.algorithm != "mark1") ) :
    raise Usage("cxi.merge "
                "d_min=4.0 "
                "data=~/scratch/r0220/006/strong/ "
                "model=3bz1_3bz2_core.pdb")
  if ((work_params.rescale_with_average_cell) and
      (not work_params.set_average_unit_cell)) :
    raise Usage("If rescale_with_average_cell=True, you must also specify "+
      "set_average_unit_cell=True.")
  if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
    raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

  log = open("%s_%s.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
  out = multi_out()
  out.register("log", log, atexit_send_to=None)
  out.register("stdout", sys.stdout)

  # Verify that the externally supplied isomorphous reference, if
  # present, defines a suitable column of intensities, and exit with
  # error if it does not.  Then warn if it is necessary to generate
  # Bijvoet mates.  Failure to catch these issues here would lead to
  # possibly obscure problems in cxi/cxi_cc.py later on.
  try:
    data_SR = mtz.object(work_params.scaling.mtz_file)
  except RuntimeError:
    pass
  else:
    array_SR = None
    obs_labels = []
    for array in data_SR.as_miller_arrays():
      this_label = array.info().label_string().lower()
      if array.observation_type() is not None:
        obs_labels.append(this_label.split(',')[0])
      if this_label.find('fobs')>=0:
        array_SR = array.as_intensity_array()
        break
      if this_label.find('imean')>=0:
        array_SR = array.as_intensity_array()
        break
      if this_label.find(work_params.scaling.mtz_column_F)==0:
        array_SR = array.as_intensity_array()
        break

    if array_SR is None:
      known_labels = ['fobs', 'imean', work_params.scaling.mtz_column_F]
      raise Usage(work_params.scaling.mtz_file +
                  " does not contain any observations labelled [" +
                  ", ".join(known_labels) +
                  "].  Please set scaling.mtz_column_F to one of [" +
                  ",".join(obs_labels) + "].")
    elif not work_params.merge_anomalous and not array_SR.anomalous_flag():
      print >> out, "Warning: Preserving anomalous contributors, but %s " \
        "has anomalous contributors merged.  Generating identical Bijvoet " \
        "mates." % work_params.scaling.mtz_file

  # Read Nat's reference model from an MTZ file.  XXX The observation
  # type is given as F, not I--should they be squared?  Check with Nat!
  print >> out, "I model"
  if work_params.model is not None:
    from xfel.merging.general_fcalc import run
    i_model = run(work_params)
    work_params.target_unit_cell = i_model.unit_cell()
    work_params.target_space_group = i_model.space_group_info()
    i_model.show_summary()
  else:
    i_model = None

  print >> out, "Target unit cell and space group:"
  print >> out, "  ", work_params.target_unit_cell
  print >> out, "  ", work_params.target_space_group

  miller_set = symmetry(
      unit_cell=work_params.target_unit_cell,
      space_group_info=work_params.target_space_group
    ).build_miller_set(
      anomalous_flag=not work_params.merge_anomalous,
      d_max=work_params.d_max,
      d_min=work_params.d_min / math.pow(
        1 + work_params.unit_cell_length_tolerance, 1 / 3))
  miller_set = miller_set.change_basis(
    work_params.model_reindex_op).map_to_asu()

  if i_model is not None:
    matches = miller.match_indices(i_model.indices(), miller_set.indices())
    assert not matches.have_singles()
    miller_set = miller_set.select(matches.permutation())

# ---- Augment this code with any special procedures for x scaling
  scaler = xscaling_manager(
    miller_set=miller_set,
    i_model=i_model,
    params=work_params,
    log=out)
  scaler.scale_all()
  if scaler.n_accepted == 0:
    return None
# --- End of x scaling
  scaler.uc_values = unit_cell_distribution()
  for icell in xrange(len(scaler.frames["unit_cell"])):
    if scaler.params.model is None:
      scaler.uc_values.add_cell(
      unit_cell=scaler.frames["unit_cell"][icell])
    else:
      scaler.uc_values.add_cell(
      unit_cell=scaler.frames["unit_cell"][icell],
      rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))

  scaler.show_unit_cell_histograms()
  if (work_params.rescale_with_average_cell) :
    average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
    average_cell = uctbx.unit_cell(list(average_cell_abc) +
      list(work_params.target_unit_cell.parameters()[3:]))
    work_params.target_unit_cell = average_cell
    print >> out, ""
    print >> out, "#" * 80
    print >> out, "RESCALING WITH NEW TARGET CELL"
    print >> out, "  average cell: %g %g %g %g %g %g" % \
      work_params.target_unit_cell.parameters()
    print >> out, ""
    scaler.reset()
    scaler = xscaling_manager(
      miller_set=miller_set,
      i_model=i_model,
      params=work_params,
      log=out)
    scaler.scale_all()
    scaler.uc_values = unit_cell_distribution()
    for icell in xrange(len(scaler.frames["unit_cell"])):
      if scaler.params.model is None:
        scaler.uc_values.add_cell(
        unit_cell=scaler.frames["unit_cell"][icell])
      else:
        scaler.uc_values.add_cell(
        unit_cell=scaler.frames["unit_cell"][icell],
        rejected=(scaler.frames["cc"][icell] < scaler.params.min_corr))
    scaler.show_unit_cell_histograms()
  if False : #(work_params.output.show_plots) :
    try :
      plot_overall_completeness(completeness)
    except Exception, e :
      print "ERROR: can't show plots"
      print "  %s" % str(e)
Exemple #32
0
def run (args) :
  verbose = False
  parser = OptionParser()
  parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
    help="Turn on verbose output")
  parser.add_option("--skip-tests", dest="skip_tests", action="store_true",
    help="Don't import modules beginning with 'tst'")
  options, args = parser.parse_args(args)
  module_list = []
  if (len(args) == 0) :
    module_list.extend([ m.name for m in libtbx.env.module_list ])
  else :
    for arg in args :
      assert (arg in libtbx.env.module_dict), arg
      module_list.append(arg)
  has_stdout = []
  stdout_old = sys.stdout
  for module_name in module_list :
    if (module_name in ignore_modules) :
      continue
    try :
      module = __import__(module_name)
    except ImportError, e :
      print >> sys.stderr, e
      continue
    assert len(module.__path__) == 1
    mod_path = module.__path__[0]
    path_fields = split_all(mod_path)
    n_leading_dirs = len(path_fields) - 1
    for dirname, dirnames, filenames in os.walk(mod_path) :
      for file_name in filenames :
        if file_name.endswith(".py") and (file_name != "libtbx_refresh.py") :
          py_mod_name, ext = op.splitext(file_name)
          if (ext != '.py') or ("." in py_mod_name) :
            if (options.verbose) :
              print >> sys.stderr, "skipping %s" % file_name
            continue
          py_path = split_all(dirname)[n_leading_dirs:]
          import_name = ".".join(py_path)
          if (not has_init_py(dirname)) or (import_name in ignore_modules) :
            continue
          top_level_module = py_path[0]
          if (file_name != "__init__.py") :
            import_name += "." + file_name[:-3]
          if (import_name in ignore_modules) :
            continue
          elif ((file_name.startswith("tst_") or file_name.startswith("test_"))
                and options.skip_tests) :
            continue
          if (options.verbose) :
            print import_name
          try :
            sys.stdout = multi_out()
            sys.stdout.register("stdout", stdout_old)
            out = StringIO()
            sys.stdout.register("stringio", out)
            submodule = __import__(import_name)
            if (out.getvalue() != '') :
              has_stdout.append(import_name)
              if (options.verbose) :
                print >> sys.stderr, out.getvalue()
          except ImportError, e :
            print >> sys.stderr, e
          finally :
            sys.stdout = stdout_old
Exemple #33
0
def run(params):
    out = multi_out()
    out.register("log", open(os.path.join(os.path.dirname(params.lstout), "multi_prep_merging.log"), "w"), atexit_send_to=None)
    out.register("stdout", sys.stdout)

    cell_params = libtbx.phil.parse(input_string=multi_check_cell_consistency.master_params_str).extract()
    cell_params.topdir = params.topdir
    cm = multi_check_cell_consistency.run(cell_params, out=out)

    if len(cm.groups) == 0:
        print "Oh, no. No data."
        return

    if params.space_group is not None and params.group_choice is None:
        params.group_choice = 1 # maybe the largest group.

    if params.group_choice < 1 or len(cm.groups) < params.group_choice:
        print "Invalid group_choice=. Should be in 1..%d" % len(cm.groups)
        return

    possible_sgs = set(map(lambda x: cm.symms[x].space_group(), cm.groups[params.group_choice-1]))
    if params.space_group is None:
        print "Please specify space_group=. The possible choice is:"
        print "\n".join(map(lambda x: x.info().symbol_and_number(), possible_sgs))
        return
    try:
        if sgtbx.space_group_info(params.space_group).group() not in possible_sgs:
            print "Invalid space group choice. The possible choice is:"
            print "\n".join(map(lambda x: x.info().symbol_and_number(), possible_sgs))
            return
    except RuntimeError:
        print "Invalid space group name or number (%s)" % params.space_group
        return

    symms = map(lambda i: cm.symms[i], cm.groups[params.group_choice-1])
    dirs = map(lambda i: cm.dirs[i], cm.groups[params.group_choice-1])

    sgnum = sgtbx.space_group_info(params.space_group).group().type().number()

    # 1. Scale with specified symmetry
    rescale_with_specified_symm(params.topdir, dirs, symms, out, sgnum=sgnum)

    # 2. Resolve reindexing problem
    # TODO: reconstruct unit cell by averaging
    reference_symm = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)[0]

    cosets = reindex.reindexing_operators(reference_symm, reference_symm)
    reidx_ops = cosets.combined_cb_ops()
    print " Reference symmetry:", reference_symm.unit_cell(), reference_symm.space_group_info().symbol_and_number(), 
    print " Possible reindex operators:", map(lambda x: str(x.as_hkl()), reidx_ops)
    if len(reidx_ops) == 1:
        print " No reindexing problem exists."
    elif params.reference_for_reindex is not None:
        resolve_indexing_ambiguity_using_reference(dirs, reidx_ops, params.reference_for_reindex)
    else:
        resolve_indexing_ambiguity(dirs, reidx_ops)

    ofs = open(params.lstout, "w")
    for wd in dirs:
        xas_full = os.path.join(wd, "XDS_ASCII_fullres.HKL")
        if os.path.isfile(xas_full): # if _fullres.HKL exists, use it.
            ofs.write("%s\n" % xas_full)
        else:
            ofs.write("%s\n" % os.path.join(wd, "XDS_ASCII.HKL"))
def run(args):
    phil = iotbx.phil.process_command_line(args=args,
                                           master_string=master_phil).show()
    work_params = phil.work.extract()
    from xfel.merging.phil_validation import application, samosa
    application(work_params)
    samosa(work_params)
    if ("--help" in args):
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or (work_params.data is None)):
        command_name = os.environ["LIBTBX_DISPATCHER_NAME"]
        raise Usage(command_name + " "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell)
            and (not work_params.set_average_unit_cell)):
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify " +
            "set_average_unit_cell=True.")
    if work_params.raw_data.sdfac_auto and work_params.raw_data.sdfac_refine:
        raise Usage("Cannot specify both sdfac_auto and sdfac_refine")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s.log" % work_params.output.prefix, "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)
    print >> out, "I model"
    if work_params.model is not None:
        from xfel.merging.general_fcalc import run
        i_model = run(work_params)
        work_params.target_unit_cell = i_model.unit_cell()
        work_params.target_space_group = i_model.space_group_info()
        i_model.show_summary()
    else:
        i_model = None

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    miller_set, i_model = consistent_set_and_model(work_params, i_model)

    frame_files = get_observations(work_params)
    scaler = scaling_manager(miller_set=miller_set,
                             i_model=i_model,
                             params=work_params,
                             log=out)
    scaler.scale_all(frame_files)
    if scaler.n_accepted == 0:
        return None
    scaler.show_unit_cell_histograms()
    if (work_params.rescale_with_average_cell):
        average_cell_abc = scaler.uc_values.get_average_cell_dimensions()
        average_cell = uctbx.unit_cell(
            list(average_cell_abc) +
            list(work_params.target_unit_cell.parameters()[3:]))
        work_params.target_unit_cell = average_cell
        print >> out, ""
        print >> out, "#" * 80
        print >> out, "RESCALING WITH NEW TARGET CELL"
        print >> out, "  average cell: %g %g %g %g %g %g" % \
          work_params.target_unit_cell.parameters()
        print >> out, ""
        scaler.reset()
        scaler.scale_all(frame_files)
        scaler.show_unit_cell_histograms()
    if False:  #(work_params.output.show_plots) :
        try:
            plot_overall_completeness(completeness)
        except Exception as e:
            print "ERROR: can't show plots"
            print "  %s" % str(e)
    print >> out, "\n"

    # Sum the observations of I and I/sig(I) for each reflection.
    sum_I = flex.double(miller_set.size(), 0.)
    sum_I_SIGI = flex.double(miller_set.size(), 0.)
    for i in xrange(miller_set.size()):
        index = miller_set.indices()[i]
        if index in scaler.ISIGI:
            for t in scaler.ISIGI[index]:
                sum_I[i] += t[0]
                sum_I_SIGI[i] += t[1]

    miller_set_avg = miller_set.customized_copy(
        unit_cell=work_params.target_unit_cell)
    table1 = show_overall_observations(obs=miller_set_avg,
                                       redundancy=scaler.completeness,
                                       summed_wt_I=scaler.summed_wt_I,
                                       summed_weight=scaler.summed_weight,
                                       ISIGI=scaler.ISIGI,
                                       n_bins=work_params.output.n_bins,
                                       title="Statistics for all reflections",
                                       out=out,
                                       work_params=work_params)
    print >> out, ""
    n_refl, corr = ((scaler.completeness > 0).count(True), 0)
    print >> out, "\n"
    table2 = show_overall_observations(
        obs=miller_set_avg,
        redundancy=scaler.summed_N,
        summed_wt_I=scaler.summed_wt_I,
        summed_weight=scaler.summed_weight,
        ISIGI=scaler.ISIGI,
        n_bins=work_params.output.n_bins,
        title="Statistics for reflections where I > 0",
        out=out,
        work_params=work_params)
    #from libtbx import easy_pickle
    #easy_pickle.dump(file_name="stats.pickle", obj=stats)
    #stats.report(plot=work_params.plot)
    #miller_counts = miller_set_p1.array(data=stats.counts.as_double()).select(
    #  stats.counts != 0)
    #miller_counts.as_mtz_dataset(column_root_label="NOBS").mtz_object().write(
    #  file_name="nobs.mtz")
    if work_params.data_subsubsets.subsubset is not None and work_params.data_subsubsets.subsubset_total is not None:
        easy_pickle.dump(
            "scaler_%d.pickle" % work_params.data_subsubsets.subsubset, scaler)
    print >> out, ""
    mtz_file, miller_array = scaler.finalize_and_save_data()
    #table_pickle_file = "%s_graphs.pkl" % work_params.output.prefix
    #easy_pickle.dump(table_pickle_file, [table1, table2])
    loggraph_file = os.path.abspath("%s_graphs.log" %
                                    work_params.output.prefix)
    f = open(loggraph_file, "w")
    f.write(table1.format_loggraph())
    f.write("\n")
    f.write(table2.format_loggraph())
    f.close()
    result = scaling_result(miller_array=miller_array,
                            plots=scaler.get_plot_statistics(),
                            mtz_file=mtz_file,
                            loggraph_file=loggraph_file,
                            obs_table=table1,
                            all_obs_table=table2,
                            n_reflections=n_refl,
                            overall_correlation=corr)
    easy_pickle.dump("%s.pkl" % work_params.output.prefix, result)
    return result
def simul_utils(args):
    if len(args) == 0:
        print_help()
    elif ("--help" in args):
        print_help()
    elif ("--h" in args):
        print_help()
    elif ("-h" in args):
        print_help()
    else:
        log = multi_out()
        if (not "--quiet" in args):
            log.register(label="stdout", file_object=sys.stdout)
        string_buffer = StringIO()
        string_buffer_plots = StringIO()
        log.register(label="log_buffer", file_object=string_buffer)

        phil_objects = []
        argument_interpreter = master_params.command_line_argument_interpreter(
            home_scope="map_coefs")

        print("#phil __OFF__", file=log)
        print("======================", file=log)
        print("          SIMUL       ", file=log)
        print("A data simulation tool", file=log)
        print("======================", file=log)
        print(file=log)

        for arg in args:
            command_line_params = None
            arg_is_processed = False
            # is it a file?
            if (os.path.isfile(arg)):  ## is this a file name?
                # check if it is a phil file
                try:
                    command_line_params = iotbx.phil.parse(file_name=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass
            else:
                try:
                    command_line_params = argument_interpreter.process(arg=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass

            if not arg_is_processed:
                print("##----------------------------------------------##",
                      file=log)
                print("## Unknown file or keyword:", arg, file=log)
                print("##----------------------------------------------##",
                      file=log)
                print(file=log)
                raise Sorry("Unknown file or keyword: %s" % arg)

        effective_params = master_params.fetch(sources=phil_objects)
        params = effective_params.extract()
        """
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    """
        # now get the unit cell from the pdb file

        hkl_xs = crystal_symmetry_from_any.extract_from(
            file_name=params.simul_utils.input.xray_data.file_name)
        pdb_xs = crystal_symmetry_from_any.extract_from(
            file_name=params.simul_utils.input.model.file_name)

        phil_xs = crystal.symmetry(
            unit_cell=params.simul_utils.input.unit_cell,
            space_group_info=params.simul_utils.input.space_group)

        combined_xs = select_crystal_symmetry(None, phil_xs, [pdb_xs],
                                              [hkl_xs])

        # inject the unit cell and symmetry in the phil scope please
        params.simul_utils.input.unit_cell = combined_xs.unit_cell()
        params.simul_utils.input.space_group = \
          sgtbx.space_group_info( group = combined_xs.space_group() )

        print("#phil __ON__", file=log)
        new_params = master_params.format(python_object=params)
        new_params.show(out=log)
        print("#phil __END__", file=log)

        if params.simul_utils.input.unit_cell is None:
            raise Sorry("unit cell not specified")
        if params.simul_utils.input.space_group is None:
            raise Sorry("space group not specified")
        if params.simul_utils.input.xray_data.file_name is None:
            raise Sorry("Xray data not specified")
        if params.simul_utils.input.model.file_name is None:
            raise Sorry("pdb file with  model not specified")

        #-----------------------------------------------------------
        #
        # step 1: read in the reflection file
        #
        phil_xs = crystal.symmetry(
            unit_cell=params.simul_utils.input.unit_cell,
            space_group_info=params.simul_utils.input.space_group)

        xray_data_server = reflection_file_utils.reflection_file_server(
            crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[])

        miller_array = None

        miller_array = xray_data_server.get_xray_data(
            file_name=params.simul_utils.input.xray_data.file_name,
            labels=params.simul_utils.input.xray_data.labels,
            ignore_all_zeros=True,
            parameter_scope='simul_utils.input.xray_data',
            parameter_name='labels')

        info = miller_array.info()

        miller_array = miller_array.map_to_asu()

        miller_array = miller_array.select(miller_array.indices() != (0, 0, 0))

        miller_array = miller_array.select(miller_array.data() > 0)
        if miller_array.sigmas() is not None:
            miller_array = miller_array.select(miller_array.sigmas() > 0)

        if (miller_array.is_xray_intensity_array()):
            miller_array = miller_array.f_sq_as_f()
        elif (miller_array.is_complex_array()):
            miller_array = abs(miller_array)

        miller_array.set_info(info)
        print(file=log)
        print("Summary info of observed data", file=log)
        print("=============================", file=log)
        miller_array.show_summary(f=log)
        print(file=log)

        free_flags = miller_array.generate_r_free_flags()

        #--------------------------------------------------------------------
        # Step 2: get an xray structure from the PDB file
        #
        model = pdb.input(file_name=params.simul_utils.input.model.file_name
                          ).xray_structure_simple(crystal_symmetry=phil_xs, )
        print("Atomic model summary", file=log)
        print("====================", file=log)
        model.show_summary()
        print(file=log)

        #-------------------------------------------------------------------
        # Step 3: make an F_model object to get model phases and amplitudes
        #
        print("Performing bulk solvent scaling", file=log)
        print("===============================", file=log)
        print(file=log)
        print(file=log)

        f_model_object = f_model.manager(f_obs=miller_array,
                                         r_free_flags=free_flags,
                                         xray_structure=model)
        f_model_object.update_all_scales(log=log)
        fmodel = abs(
            f_model_object.f_model()).set_observation_type(miller_array)

        mockfmodel = None
        if params.simul_utils.input.mock_model.file_name is not None:
            print("Reading in mock model", file=log)
            print("=====================", file=log)
            print(file=log)
            print(file=log)
            mock_model = pdb.input(file_name=params.simul_utils.input.
                                   mock_model.file_name).xray_structure_simple(
                                       crystal_symmetry=phil_xs)
            mock_f_model = f_model.manager(f_obs=miller_array,
                                           r_free_flags=free_flags,
                                           xray_structure=mock_model)

            mock_f_model.update(k_sol=f_model_object.k_sol(),
                                b_sol=f_model_object.b_sol(),
                                b_cart=f_model_object.b_cart())
            mockfmodel = abs(
                mock_f_model.f_model()).set_observation_type(miller_array)
        else:
            mockfmodel = fmodel.deep_copy()

        print("Making new data", file=log)
        print("===============", file=log)
        print(file=log)
        print(file=log)

        new_data_builder = error_swap(miller_array, fmodel, mockfmodel)
        new_data = new_data_builder.new_obs
        # we now have to write the data actually

        print("Writing new data set", file=log)
        print("====================", file=log)

        mtz_dataset = new_data.as_mtz_dataset(column_root_label="FOBS")
        mtz_dataset.mtz_object().write(
            file_name=params.simul_utils.output.hklout)
Exemple #36
0
    def run(self, args, command_name, out=sys.stdout):
        command_line = (iotbx_option_parser(
            usage="%s [options]" % command_name,
            description='Example: %s data.mtz data.mtz ref_model.pdb' %
            command_name).option(
                None,
                "--show_defaults",
                action="store_true",
                help="Show list of parameters.")).process(args=args)

        cif_file = None
        processed_args = utils.process_command_line_args(
            args=args, log=sys.stdout, master_params=master_phil)
        params = processed_args.params
        if (params is None): params = master_phil
        self.params = params.extract().ensemble_probability
        pdb_file_names = processed_args.pdb_file_names
        if len(pdb_file_names) != 1:
            raise Sorry("Only one PDB structure may be used")
        pdb_file = file_reader.any_file(pdb_file_names[0])
        self.log = multi_out()
        self.log.register(label="stdout", file_object=sys.stdout)
        self.log.register(label="log_buffer",
                          file_object=StringIO(),
                          atexit_send_to=None)
        sys.stderr = self.log
        log_file = open(
            pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
            '_pensemble.log', "w")

        self.log.replace_stringio(old_label="log_buffer",
                                  new_label="log",
                                  new_file_object=log_file)
        utils.print_header(command_name, out=self.log)
        params.show(out=self.log)
        #
        f_obs = None
        r_free_flags = None
        reflection_files = processed_args.reflection_files

        if self.params.fobs_vs_fcalc_post_nll:
            if len(reflection_files) == 0:
                raise Sorry(
                    "Fobs from input MTZ required for fobs_vs_fcalc_post_nll")

        if len(reflection_files) > 0:
            crystal_symmetry = processed_args.crystal_symmetry
            print('Reflection file : ',
                  processed_args.reflection_file_names[0],
                  file=self.log)
            utils.print_header("Model and data statistics", out=self.log)
            rfs = reflection_file_server(
                crystal_symmetry=crystal_symmetry,
                reflection_files=processed_args.reflection_files,
                log=self.log)

            parameters = extract_xtal_data.data_and_flags_master_params(
            ).extract()
            determine_data_and_flags_result = extract_xtal_data.run(
                reflection_file_server=rfs,
                parameters=parameters,
                data_parameter_scope="refinement.input.xray_data",
                flags_parameter_scope="refinement.input.xray_data.r_free_flags",
                data_description="X-ray data",
                keep_going=True,
                log=self.log)
            f_obs = determine_data_and_flags_result.f_obs
            number_of_reflections = f_obs.indices().size()
            r_free_flags = determine_data_and_flags_result.r_free_flags
            test_flag_value = determine_data_and_flags_result.test_flag_value
            if (r_free_flags is None):
                r_free_flags = f_obs.array(
                    data=flex.bool(f_obs.data().size(), False))

        # process PDB
        pdb_file.assert_file_type("pdb")
        #
        pdb_in = hierarchy.input(file_name=pdb_file.file_name)
        ens_pdb_hierarchy = pdb_in.construct_hierarchy()
        ens_pdb_hierarchy.atoms().reset_i_seq()
        ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
        number_structures = len(ens_pdb_xrs_s)
        print('Number of structure in ensemble : ',
              number_structures,
              file=self.log)

        # Calculate sigmas from input map only
        if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
            # process MTZ
            input_file = file_reader.any_file(
                self.params.ensemble_sigma_map_input)
            if input_file.file_type == "hkl":
                if input_file.file_object.file_type() != "ccp4_mtz":
                    raise Sorry("Only MTZ format accepted for map input")
                else:
                    mtz_file = input_file
            else:
                raise Sorry("Only MTZ format accepted for map input")
            miller_arrays = mtz_file.file_server.miller_arrays
            map_coeffs_1 = miller_arrays[0]
            #
            xrs_list = []
            for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
                # get sigma levels from ensemble fc for each structure
                xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy,
                                    ens_pdb_xrs=ens_pdb_xrs,
                                    map_coeffs_1=map_coeffs_1,
                                    residue_detail=self.params.residue_detail,
                                    ignore_hd=self.params.ignore_hd,
                                    log=self.log)
                xrs_list.append(xrs)
            # write ensemble pdb file, occupancies as sigma level
            filename = pdb_file_names[0].split('/')[-1].replace(
                '.pdb',
                '') + '_vs_' + self.params.ensemble_sigma_map_input.replace(
                    '.mtz', '') + '_pensemble.pdb'
            write_ensemble_pdb(filename=filename,
                               xrs_list=xrs_list,
                               ens_pdb_hierarchy=ens_pdb_hierarchy)

        # Do full analysis vs Fobs
        else:
            model_map_coeffs = []
            fmodel = None
            # Get <fcalc>
            for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
                ens_pdb_xrs.set_occupancies(1.0)
                if model == 0:
                    # If mtz not supplied get fobs from xray structure...
                    # Use input Fobs for scoring against nll
                    if self.params.fobs_vs_fcalc_post_nll:
                        dummy_fobs = f_obs
                    else:
                        if f_obs == None:
                            if self.params.fcalc_high_resolution == None:
                                raise Sorry(
                                    "Please supply high resolution limit or input mtz file."
                                )
                            dummy_dmin = self.params.fcalc_high_resolution
                            dummy_dmax = self.params.fcalc_low_resolution
                        else:
                            print(
                                'Supplied mtz used to determine high and low resolution cuttoffs',
                                file=self.log)
                            dummy_dmax, dummy_dmin = f_obs.d_max_min()
                        #
                        dummy_fobs = abs(
                            ens_pdb_xrs.structure_factors(
                                d_min=dummy_dmin).f_calc())
                        dummy_fobs.set_observation_type_xray_amplitude()
                        # If mtz supplied, free flags are over written to prevent array size error
                        r_free_flags = dummy_fobs.array(
                            data=flex.bool(dummy_fobs.data().size(), False))
                    #
                    fmodel = utils.fmodel_simple(
                        scattering_table="wk1995",
                        xray_structures=[ens_pdb_xrs],
                        f_obs=dummy_fobs,
                        target_name='ls',
                        bulk_solvent_and_scaling=False,
                        r_free_flags=r_free_flags)
                    f_calc_ave = fmodel.f_calc().array(
                        data=fmodel.f_calc().data() * 0).deep_copy()
                    # XXX Important to ensure scale is identical for each model and <model>
                    fmodel.set_scale_switch = 1.0
                    f_calc_ave_total = fmodel.f_calc().data().deep_copy()
                else:
                    fmodel.update_xray_structure(xray_structure=ens_pdb_xrs,
                                                 update_f_calc=True,
                                                 update_f_mask=False)
                    f_calc_ave_total += fmodel.f_calc().data().deep_copy()
                print('Model :', model + 1, file=self.log)
                print("\nStructure vs real Fobs (no bulk solvent or scaling)",
                      file=self.log)
                print('Rwork          : %5.4f ' % fmodel.r_work(),
                      file=self.log)
                print('Rfree          : %5.4f ' % fmodel.r_free(),
                      file=self.log)
                print('K1             : %5.4f ' % fmodel.scale_k1(),
                      file=self.log)
                fcalc_edm = fmodel.electron_density_map()
                fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc')
                fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(
                    column_root_label='Fc')
                if self.params.output_model_and_model_ave_mtz:
                    fcalc_mtz_dataset.mtz_object().write(
                        file_name=str(model + 1) + "_Fc.mtz")
                model_map_coeffs.append(fcalc_map_coeffs.deep_copy())

            fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total /
                                                  number_structures))
            print("\nEnsemble vs real Fobs (no bulk solvent or scaling)",
                  file=self.log)
            print('Rwork          : %5.4f ' % fmodel.r_work(), file=self.log)
            print('Rfree          : %5.4f ' % fmodel.r_free(), file=self.log)
            print('K1             : %5.4f ' % fmodel.scale_k1(), file=self.log)

            # Get <Fcalc> map
            fcalc_ave_edm = fmodel.electron_density_map()
            fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(
                map_type='Fc').deep_copy()
            fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(
                column_root_label='Fc')
            if self.params.output_model_and_model_ave_mtz:
                fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz")
            fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
            fcalc_ave_map_coeffs.apply_volume_scaling()
            fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
            fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)

            print("<Fcalc> Map Stats :", file=self.log)
            fcalc_ave_map_stats.show_summary(f=self.log)
            offset = fcalc_ave_map_stats.min()
            model_neg_ll = []

            number_previous_scatters = 0

            # Run through structure list again and get probability
            xrs_list = []
            for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
                if self.params.verbose:
                    print('\n\nModel                   : ',
                          model + 1,
                          file=self.log)
                # Get model atom sigmas vs Fcalc
                fcalc_map = model_map_coeffs[model].fft_map()
                fcalc_map.apply_volume_scaling()
                fcalc_map_data = fcalc_map.real_map_unpadded()
                fcalc_map_stats = maptbx.statistics(fcalc_map_data)
                if self.params.verbose:
                    print("Fcalc map stats         :", file=self.log)
                fcalc_map_stats.show_summary(f=self.log)

                xrs = get_map_sigma(
                    ens_pdb_hierarchy=ens_pdb_hierarchy,
                    ens_pdb_xrs=ens_pdb_xrs,
                    fft_map_1=fcalc_map,
                    model_i=model,
                    residue_detail=self.params.residue_detail,
                    ignore_hd=self.params.ignore_hd,
                    number_previous_scatters=number_previous_scatters,
                    log=self.log)
                fcalc_sigmas = xrs.scatterers().extract_occupancies()
                del fcalc_map
                # Get model atom sigmas vs <Fcalc>
                xrs = get_map_sigma(
                    ens_pdb_hierarchy=ens_pdb_hierarchy,
                    ens_pdb_xrs=ens_pdb_xrs,
                    fft_map_1=fcalc_ave_map_coeffs,
                    model_i=model,
                    residue_detail=self.params.residue_detail,
                    ignore_hd=self.params.ignore_hd,
                    number_previous_scatters=number_previous_scatters,
                    log=self.log)

                ### For testing other residue averaging options
                #print xrs.residue_selections

                fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
                # Probability of model given <model>
                prob = fcalc_ave_sigmas / fcalc_sigmas
                # XXX debug option
                if False:
                    for n, p in enumerate(prob):
                        print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log)
                # Set probabilty between 0 and 1
                # XXX Make Histogram / more stats
                prob_lss_zero = flex.bool(prob <= 0)
                prob_grt_one = flex.bool(prob > 1)
                prob.set_selected(prob_lss_zero, 0.001)
                prob.set_selected(prob_grt_one, 1.0)
                xrs.set_occupancies(prob)
                xrs_list.append(xrs)
                sum_neg_ll = sum(-flex.log(prob))
                model_neg_ll.append((sum_neg_ll, model))
                if self.params.verbose:
                    print('Model probability stats :', file=self.log)
                    print(prob.min_max_mean().show(), file=self.log)
                    print('  Count < 0.0 : ',
                          prob_lss_zero.count(True),
                          file=self.log)
                    print('  Count > 1.0 : ',
                          prob_grt_one.count(True),
                          file=self.log)

                # For averaging by residue
                number_previous_scatters += ens_pdb_xrs.sites_cart().size()

            # write ensemble pdb file, occupancies as sigma level
            write_ensemble_pdb(
                filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
                '_pensemble.pdb',
                xrs_list=xrs_list,
                ens_pdb_hierarchy=ens_pdb_hierarchy)

            # XXX Test ordering models by nll
            # XXX Test removing nth percentile atoms
            if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
                for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]:
                    model_neg_ll = sorted(model_neg_ll)
                    f_calc_ave_total_reordered = None
                    print_list = []
                    for i_neg_ll in model_neg_ll:
                        xrs = xrs_list[i_neg_ll[1]]
                        nll_occ = xrs.scatterers().extract_occupancies()

                        # Set q=0 nth percentile atoms
                        sorted_nll_occ = sorted(nll_occ, reverse=True)
                        number_atoms = len(sorted_nll_occ)
                        percentile_prob_cutoff = sorted_nll_occ[
                            int(number_atoms * percentile) - 1]
                        cutoff_selections = flex.bool(
                            nll_occ < percentile_prob_cutoff)
                        cutoff_nll_occ = flex.double(nll_occ.size(),
                                                     1.0).set_selected(
                                                         cutoff_selections,
                                                         0.0)
                        #XXX Debug
                        if False:
                            print('\nDebug')
                            for x in range(len(cutoff_selections)):
                                print(cutoff_selections[x], nll_occ[x],
                                      cutoff_nll_occ[x])
                            print(percentile)
                            print(percentile_prob_cutoff)
                            print(cutoff_selections.count(True))
                            print(cutoff_selections.size())
                            print(cutoff_nll_occ.count(0.0))
                            print('Count q = 1           : ',
                                  cutoff_nll_occ.count(1.0))
                            print('Count scatterers size : ',
                                  cutoff_nll_occ.size())

                        xrs.set_occupancies(cutoff_nll_occ)
                        fmodel.update_xray_structure(xray_structure=xrs,
                                                     update_f_calc=True,
                                                     update_f_mask=True)

                        if f_calc_ave_total_reordered == None:
                            f_calc_ave_total_reordered = fmodel.f_calc().data(
                            ).deep_copy()
                            f_mask_ave_total_reordered = fmodel.f_masks(
                            )[0].data().deep_copy()
                            cntr = 1
                        else:
                            f_calc_ave_total_reordered += fmodel.f_calc().data(
                            ).deep_copy()
                            f_mask_ave_total_reordered += fmodel.f_masks(
                            )[0].data().deep_copy()
                            cntr += 1
                        fmodel.update(
                            f_calc=f_calc_ave.array(
                                f_calc_ave_total_reordered / cntr).deep_copy(),
                            f_mask=f_calc_ave.array(
                                f_mask_ave_total_reordered / cntr).deep_copy())

                        # Update solvent and scale
                        # XXX Will need to apply_back_trace on latest version
                        fmodel.set_scale_switch = 0
                        fmodel.update_all_scales()

                        # Reset occ for outout
                        xrs.set_occupancies(nll_occ)
                        # k1 updated vs Fobs
                        if self.params.fobs_vs_fcalc_post_nll:
                            print_list.append([
                                cntr, i_neg_ll[0], i_neg_ll[1],
                                fmodel.r_work(),
                                fmodel.r_free()
                            ])

                    # Order models by nll and print summary
                    print(
                        '\nModels ranked by nll <Fcalc> R-factors recalculated',
                        file=self.log)
                    print('Percentile cutoff : {0:5.3f}'.format(percentile),
                          file=self.log)
                    xrs_list_sorted_nll = []
                    print('      |      NLL     <Rw>     <Rf>    Ens Model',
                          file=self.log)
                    for info in print_list:
                        print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.
                              format(
                                  info[0],
                                  info[1],
                                  info[3],
                                  info[4],
                                  info[2] + 1,
                              ),
                              file=self.log)
                        xrs_list_sorted_nll.append(xrs_list[info[2]])

                # Output nll ordered ensemble

                write_ensemble_pdb(
                    filename='nll_ordered_' +
                    pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
                    '_pensemble.pdb',
                    xrs_list=xrs_list_sorted_nll,
                    ens_pdb_hierarchy=ens_pdb_hierarchy)
Exemple #37
0
def run (args,
    out=sys.stdout,
    program_name="phenix.molprobity",
    ignore_missing_modules=False,
    return_input_objects=False) : # for testing
  rotarama_dir = libtbx.env.find_in_repositories(
    relative_path="chem_data/rotarama_data",
    test=os.path.isdir)
  if (rotarama_dir is None) :
    raise ImportError("Rotamer and Ramachandran distributions not available; "+
      "you will need these to run MolProbity.")
  elif (((not libtbx.env.has_module("reduce")) or
         (not libtbx.env.has_module("probe"))) and
         (not ignore_missing_modules)) :
    raise ImportError("Reduce and/or Probe not configured.")
  import mmtbx.validation.molprobity
  import mmtbx.command_line
  cmdline = mmtbx.command_line.load_model_and_data(
    args=args,
    master_phil=get_master_phil(),
    require_data=False,
    create_fmodel=True,
    process_pdb_file=True,
    usage_string=usage_string,
    prefer_anomalous=True,
    out=out)
  params = cmdline.params
  fmodel = cmdline.fmodel
  if (params.output.maps is Auto) and (fmodel is not None) :
    params.output.maps = True
  elif (params.output.maps == True) and (fmodel is None) :
    raise Sorry("Map output requires experimental data.")
  if (params.molprobity.keep_hydrogens is Auto) :
    params.molprobity.keep_hydrogens = \
      (params.input.scattering_table == "neutron")
  header_info = mmtbx.validation.molprobity.pdb_header_info(
    pdb_file=params.input.pdb.file_name[0],
    pdb_hierarchy=cmdline.pdb_hierarchy)
  pdb_prefix = os.path.splitext(os.path.basename(
    params.input.pdb.file_name[0]))[0]
  if (params.output.prefix is None) :
    params.output.prefix = "molprobity"
  probe_file = None
  if (params.output.probe_dots) or (params.output.kinemage) :
    probe_file = params.output.prefix + "_probe.txt"
  raw_data = cmdline.raw_data

  # check map parameters
  from mmtbx.real_space_correlation import check_map_file
  check_map_file(None, params.input.maps)

  validation = mmtbx.validation.molprobity.molprobity(
    pdb_hierarchy=cmdline.pdb_hierarchy,
    xray_structure=cmdline.xray_structure,
    fmodel=fmodel,
    flags=params.molprobity.flags,
    crystal_symmetry=cmdline.crystal_symmetry,
    sequences=cmdline.sequence,
    geometry_restraints_manager=cmdline.geometry,
    raw_data=cmdline.raw_data,
    unmerged_data=cmdline.unmerged_i_obs,
    all_chain_proxies=cmdline.processed_pdb_file.all_chain_proxies,
    header_info=header_info,
    keep_hydrogens=params.molprobity.keep_hydrogens,
    nuclear=params.molprobity.nuclear,
    save_probe_unformatted_file=probe_file,
    min_cc_two_fofc=params.molprobity.min_cc_two_fofc,
    n_bins_data=params.molprobity.n_bins,
    outliers_only=params.molprobity.outliers_only,
    use_pdb_header_resolution_cutoffs=\
      params.molprobity.use_pdb_header_resolution_cutoffs,
    count_anomalous_pairs_separately=\
      params.molprobity.count_anomalous_pairs_separately,
    use_internal_variance=params.input.unmerged_data.use_internal_variance,
    file_name=params.input.pdb.file_name[0],
    ligand_selection=params.molprobity.ligand_selection,
    rotamer_library=params.molprobity.rotamer_library,
    map_params=params)
  map_file = None
  if (not params.output.quiet) :
    out2 = multi_out()
    out2.register("stdout", out)
    f = open(params.output.prefix + ".out", "w")
    out2.register("txt_out", f)
    validation.show(out=out2,
      outliers_only=params.molprobity.outliers_only,
      show_percentiles=params.output.percentiles)
    f.close()
    print >> out, ""
    print >> out, "Results written to %s.out" % params.output.prefix
    if (params.output.kinemage) :
      if (cmdline.pdb_hierarchy.models_size() == 1) :
        assert (probe_file is not None)
        import mmtbx.kinemage.validation
        kin_file = "%s.kin" % params.output.prefix
        kin_out = \
          mmtbx.kinemage.validation.export_molprobity_result_as_kinemage(
            result=validation,
            pdb_hierarchy=cmdline.pdb_hierarchy,
            geometry=cmdline.geometry,
            probe_file=probe_file,
            keep_hydrogens=params.molprobity.keep_hydrogens,
            pdbID=pdb_prefix)
        f = open(kin_file, "w")
        f.write(kin_out)
        f.close()
        if (not params.output.quiet) :
          print >> out, "Wrote kinemage to %s" % kin_file
      else :
        print >> out, "Kinemage output not available for multiple MODELs."
    if (params.output.pickle) :
      easy_pickle.dump("%s.pkl" % params.output.prefix, validation)
      if (not params.output.quiet) :
        print >> out, "Saved result to %s.pkl" % params.output.prefix
    if (params.output.coot) :
      coot_file = "%s_coot.py" % params.output.prefix
      validation.write_coot_script(coot_file)
      if (not params.output.quiet) :
        print >> out, "Wrote script for Coot: %s" % coot_file
    if (params.output.maps == True) :
      import mmtbx.maps.utils
      import iotbx.map_tools
      map_file = "%s_maps.mtz" % params.output.prefix
      two_fofc_map, fofc_map = mmtbx.maps.utils.get_maps_from_fmodel(
        fmodel=fmodel,
        fill_missing_f_obs=params.output.map_options.fill_missing_f_obs,
        exclude_free_r_reflections=\
          params.output.map_options.exclude_free_r_reflections)
      anom_map = None
      if (fmodel.f_obs().anomalous_flag()) :
        anom_map = mmtbx.maps.utils.get_anomalous_map(fmodel)
      iotbx.map_tools.write_map_coeffs(
        file_name=map_file,
        fwt_coeffs=two_fofc_map,
        delfwt_coeffs=fofc_map,
        anom_coeffs=anom_map)
      print >> out, "Wrote map coefficients to %s" % map_file
  else :
    print >> out, ""
    validation.show_summary(out=out, show_percentiles=params.output.percentiles)
  if (params.output.wxplots) :
    try :
      import wxtbx.app
    except ImportError, e :
      raise Sorry("wxPython not available.")
    else :
      app = wxtbx.app.CCTBXApp(0)
      validation.display_wx_plots()
      app.MainLoop()
Exemple #38
0
def run(params, topdirs):
    import sys

    LogClass = dict(sp8=BssJobLog,
                    pf=None)

    out_root = os.path.dirname(params.pklout)
    out = multi_out()
    out.register("log", open(os.path.join(out_root, "find_datasets.log"), "w"), atexit_send_to=None)
    out.register("stdout", sys.stdout)

    if params.find_spots:
        shikalog = open(os.path.join(out_root, "shika.log"), "w")
        shikalog.write("prefix idx nspots\n")

    config_mgr = spot_finder_gui.ConfigManager(use_cuda=params.use_cuda)

    logobjects = []

    for topdir in topdirs:
        print >>out, "Looking into %s\n" % topdir
        for root, dirnames, filenames in os.walk(topdir, followlinks=True):
            logs = filter(lambda x: x.endswith(".log"), filenames)
            for log in logs:
                log = os.path.join(root, log)

                for logtype in params.logtype:
                    if 1:#try:
                        try:
                            logobj = LogClass[logtype](log)
                        except:
                            print >>out, traceback.format_exc()
                            print >>out, "\nException raised when parsing %s\n"%log
                            raise

                        logobj.jobs = filter(lambda x: x.job_mode != "XAFS", logobj.jobs)
                        if len(logobj.jobs) > 0:
                            print >>out, "Found job log:", log
                            spots = []
                            for job in logobj.jobs:
                                if params.find_spots:
                                    spots.append(search_spots(os.path.join(root, os.path.basename(job.filename)),
                                                              job.n_images, config_mgr))
                                    for idx, f, stat in spots[-1]:
                                        if stat is None: continue
                                        n_spots = stat.spots.get_n_spots("hi_pass_resolution_spots")
                                        shikalog.write("%s %6d %4d\n" % (os.path.join(root,
                                                                                      os.path.basename(job.filename)),
                                                                         idx, n_spots))
                                        shikalog.flush()
                                else:
                                    spots.append([])
                            logobjects.append([log, logobj, spots])
                    #except:
                    #    pass

    print >>out

    logobjects.sort(key=lambda x:x[0])

    for log, logobj, spots in logobjects:
        print log
        for job, spt in zip(logobj.jobs, spots):
            jobtype = "?"
            if job.advanced_centering == {} or test_gonio_coords_equal(job.advanced_centering.get("centers",[])):
                jobtype = "Single  " 
            elif job.advanced_centering.get("mode", "") == "vector_centering":
                jobtype = "Helical "
            elif job.advanced_centering.get("mode", "") == "multiple_centering":
                jobtype = "MultiCen"
            else:
                print >>out, "WARNING:: WHY REACH HERE?", job.advanced_centering
                
            osc_range = job.osc_end - job.osc_start
            #nfound = check_files_exist(job.filename, job.n_images, os.path.dirname(log))
            nfound = len(dataset.find_existing_files_in_template(job.filename, 1, job.n_images,
                                                                 datadir=os.path.dirname(log),
                                                                 check_compressed=True))

            print >>out, " %s osc_step=%6.3f osc_range=%5.1f found=%d/%d %s beam_size=%s" % (job.beamline, job.osc_step, osc_range,
                                                   nfound, job.n_images, jobtype,
                                                   "x".join(map(lambda x:"%.1f"%x,job.beam_size)))
            if params.find_spots:
                n_spots = [stat.spots.get_n_spots("hi_pass_resolution_spots") for idx, f, stat in spt if stat is not None]
                gt20 = len(filter(lambda x: x>=20, n_spots))
                print >>out, " spots < 5A: %d..%d (>=20 spots: %d frames)" % (min(n_spots), max(n_spots), gt20)

        print >>out

    pickle.dump(logobjects, open(params.pklout, "w"), -1)
def run(args, command_name="phenix.explore_metric_symmetry"):
  command_line = (
    option_parser(
    usage=command_name+" [options]",
    description="""\
Explore Metric Symmetry. A list of possible unit cells and spacegroups is
given for the given specified unit cell and spacegroup combination. If a
second unit cell is given, linear combinations of the basis vector of one
unit cell are sought that match the other.""")

    .enable_symmetry_comprehensive()

    .option(None, "--max_delta",
            action = "store",
            type="float",
            default=5.0,
            dest = "max_delta",
            help = "Maximum delta/obliquity used in determining the lattice symmetry, using a modified Le-Page algorithm. Default is 5.0 degrees",
            metavar="FLOAT")

    .option(None, "--start_from_p1",
            action="store_true",
            dest="niggli",
            default=False,
            help="Reduce to Niggli cell and forget the input spacegroup before higher metric symmetry is sought.")

    .option(None, "--graph",
            action="store",
            default=None,
            help="A graphical representation of the graph will be written out."
                 " Requires Graphviz to be installed and on PATH.")

    .option(None, "--centring_type",
            action="store",
            type="str",
            help="Centring type, choose from P,A,B,C,I,R,F")

    .option(None, "--other_unit_cell",
            action="store",
            type="str",
            help="Other unit cell, for unit cell comparison",
            metavar="10,20,30,90,103.7,90")

    .option(None, "--other_space_group",
            action="store",
            type="str",
            help="space group for other_unit_cell, for unit cell comparison")

    .option(None, "--other_centring_type",
            action="store",
            type="str",
            help="Centring type, choose from P,A,B,C,I,R,F")

    .option(None, "--no_point_group_graph",
            action="store_true",
            dest="pg_graph",
            default=False,
            help="Do not carry out the construction of a point group graph." )

    .option(None, "--relative_length_tolerance",
            action="store",
            type="float",
            help="Tolerance for unit cell lengths to be considered equal-ish.",
            default=0.10,
            metavar="FLOAT",
            dest="rel_length_tol")

    .option(None, "--absolute_angle_tolerance",
            action="store",
            dest="abs_angle_tol",
            type="float",
            default=10.0,
            metavar="FLOAT",
            help="Angular tolerance in unit cell comparison")

     .option(None, "--max_order",
             action="store",
             type="int",
             default=1,
             metavar="INT",
             help="Maximum volume change for target cell" )
    ).process(args=args)

  log = multi_out()
  log.register(label="stdout", file_object=sys.stdout)

  allowed_centring_types={"P":"Primitive",
                          "A":"A centered",
                          "B":"B centered",
                          "C":"C centered",
                          "I":"Body centered",
                          "R":"Rombohedral",
                          "F":"Face centered"}
  if command_line.options.centring_type is not None:
    if not allowed_centring_types.has_key( command_line.options.centring_type ):
      print >> log, "Sorry, the centring type %s is not known."%(command_line.options.centring_type)
      print >> log, "Choose from P,A,B,C,I,R,F "
      return

  xs = None
  other_xs = None

  if len(args)==0:
    command_line.parser.show_help()
    return

  if ( command_line.symmetry.unit_cell() == None ):
    print >> log
    print >> log, "Sorry: Unit cell not specified."
    print >> log
    command_line.parser.show_help()
    return

  if command_line.options.centring_type is None:
    if ( command_line.symmetry.space_group_info() == None ):
      print>> log
      print>> log,  "Sorry: centring type or space group not specified."
      print>> log
      command_line.parser.show_help()
      return
  if command_line.symmetry.space_group_info()  is not None:
    if not ( command_line.symmetry.space_group().is_chiral() ):
      print >> log, "Sorry, Non chiral space groups not yet supported."
      return

  if command_line.options.centring_type is not None:
    xs  = crystal.symmetry(
      unit_cell=command_line.symmetry.unit_cell(),
      space_group_symbol="Hall: %s 1" %( command_line.options.centring_type )
      )
    command_line.symmetry = xs

  if command_line.options.niggli:
    print >> log, "*Unit cell will be niggli reduced and P1 will be assumed*"
    uc = command_line.symmetry.change_basis(
      command_line.symmetry.change_of_basis_op_to_niggli_cell() ).unit_cell()
    command_line.symmetry = crystal.symmetry( uc, "P 1" )

  xs = command_line.symmetry

  ############################################################################
  # ABOVE IS JUST INPUT PARSING, NOW THE ACTUAL STUFF HAPPENS
  ############################################################################


  if not command_line.options.pg_graph:
    ##############################
    #   get a point group graph  #
    ##############################

    pg_object = do_pointgroup_tricks( xs.unit_cell(),
                                      xs.space_group(),
                                      command_line.options.max_delta,
                                      log )

    ################################################
    #  make a graphical representation if desired  #
    ################################################

    if command_line.options.graph is not None:
      make_graph_of_graph(pg_object,
                          command_line.options.graph,
                          log)


  #########################################
  #  Check if other cell has been defined #
  #########################################

  if command_line.options.other_unit_cell is not None:
    print >> log, "A second unit cell has been specified. "
    other_xs = None

    if command_line.options.other_space_group is None:
      if command_line.options.other_centring_type is None:
        raise Sorry("No space group or centring type for other cell specified.")
      else:
        other_xs = crystal.symmetry( command_line.options.other_unit_cell,
                                     space_group_symbol="Hall: %s 1" %( command_line.options.other_centring_type )
                                   )
    else:
      other_xs = crystal.symmetry( command_line.options.other_unit_cell,
                                   space_group_symbol=command_line.options.other_space_group
                                 )

    # get the graph is desired
    if not command_line.options.pg_graph:
      other_pg_object = do_pointgroup_tricks( other_xs.unit_cell(),
                                              other_xs.space_group(),
                                              command_line.options.max_delta,
                                              log )
    # do the unit cell comparison
    print >> log
    print >> log
    print >> log, "Unit cell comparison"
    print >> log, "--------------------"
    print >> log
    print >> log, "The unit cells will be compared. The smallest niggli cell,"
    print >> log, "will be used as a (semi-flexible) lego-block to see if it"
    print >> log, "can construct the larger Niggli cell."
    print >> log
    print >> log

    order = command_line.options.max_order

    if order==1:
      sl_object =  slt.compare_lattice(xs_a=xs,
                                       xs_b=other_xs,
                                       max_delta=command_line.options.max_delta,
                                       out=log,
                                       relative_length_tolerance=command_line.options.rel_length_tol,
                                       absolute_angle_tolerance=command_line.options.abs_angle_tol)
    else:

      tmp_a = xs.change_basis( xs.change_of_basis_op_to_niggli_cell() )
      tmp_b = other_xs.change_basis( other_xs.change_of_basis_op_to_niggli_cell() )
      modified_xs = None
      order = command_line.options.max_order
      lego_block = None
      if ( tmp_a.unit_cell().volume() > tmp_b.unit_cell().volume() ):
        modified_xs = slt.make_list_of_target_xs_up_to_order( xs, order )
        lego_block = other_xs
      else:
        modified_xs = slt.make_list_of_target_xs_up_to_order( other_xs, order )
        lego_block = xs

      print >> log
      print >> log, "Volume change of largest niggli cell requested via keyword --max_order"
      print >> log
      print >> log, "Input crystal symmetry is tranformed to niggli setting using the operator:"
      print >> log,  modified_xs.basic_to_niggli_cb_op.as_xyz()
      print >> log
      print >> log, "Comparisons for various sublattices of the target cell are listed"
      print >> log

      for tmp_xs,cb_op,mat in zip(modified_xs.xs_list,
                                  modified_xs.extra_cb_op,
                                  modified_xs.matrices ):
        mat=mat.as_list_of_lists()
        print >> log, "==================================================================="
        print >> log, "Niggli cell is expanded using matrix:"
        print >> log
        print >> log, "               /%4i %4i %4i  \  "%(mat[0][0],mat[0][1],mat[0][2])
        print >> log, "          M =  |%4i %4i %4i  |  "%(mat[1][0],mat[1][1],mat[1][2])
        print >> log, "               \%4i %4i %4i  /  "%(mat[2][0],mat[2][1],mat[2][2])
        print >> log
        print >> log, "Change of basis operator to reference setting:"
        print >> log, "    ", cb_op.as_xyz()
        print >> log, "resulting crystal symmetry:"
        tmp_xs.show_summary(f=log,prefix="   ")
        print >> log
        print >> log
        sl_object =  slt.compare_lattice(xs_a=tmp_xs,
                                         xs_b=lego_block,
                                         max_delta=command_line.options.max_delta,
                                         out=log,
                                         relative_length_tolerance=command_line.options.rel_length_tol,
                                         absolute_angle_tolerance=command_line.options.abs_angle_tol)
Exemple #40
0
 def __init__ (self,
     args,
     master_phil,
     out=sys.stdout,
     process_pdb_file=True,
     require_data=True,
     create_fmodel=True,
     prefer_anomalous=None,
     force_non_anomalous=False,
     set_wavelength_from_model_header=False,
     set_inelastic_form_factors=None,
     usage_string=None,
     create_log_buffer=False,
     remove_unknown_scatterers=False,
     generate_input_phil=False) :
   import mmtbx.monomer_library.pdb_interpretation
   import mmtbx.monomer_library.server
   import mmtbx.utils
   from iotbx import crystal_symmetry_from_any
   from iotbx import file_reader
   import iotbx.phil
   if generate_input_phil :
     assert isinstance(master_phil, basestring)
     master_phil = generate_master_phil_with_inputs(phil_string=master_phil)
   if isinstance(master_phil, str) :
     master_phil = iotbx.phil.parse(master_phil)
   if (usage_string is not None) :
     if (len(args) == 0) or ("--help" in args) :
       raise Usage("""%s\n\nFull parameters:\n%s""" % (usage_string,
         master_phil.as_str(prefix="  ")))
   if (force_non_anomalous) :
     assert (not prefer_anomalous)
   assert (set_inelastic_form_factors in [None, "sasaki", "henke"])
   self.args = args
   self.master_phil = master_phil
   self.processed_pdb_file = self.pdb_inp = None
   self.pdb_hierarchy = self.xray_structure = None
   self.geometry = None
   self.sequence = None
   self.fmodel = None
   self.f_obs = None
   self.r_free_flags = None
   self.intensity_flag = None
   self.raw_data = None
   self.raw_flags = None
   self.test_flag_value = None
   self.miller_arrays = None
   self.hl_coeffs = None
   self.cif_objects = []
   self.log = out
   if ("--quiet" in args) or ("quiet=True" in args) :
     self.log = null_out()
   elif create_log_buffer :
     self.log = multi_out()
     self.log.register(label="stdout", file_object=out)
     self.log.register(label="log_buffer", file_object=StringIO())
   make_header("Collecting inputs", out=self.log)
   cmdline = iotbx.phil.process_command_line_with_files(
     args=args,
     master_phil=master_phil,
     pdb_file_def="input.pdb.file_name",
     reflection_file_def="input.xray_data.file_name",
     cif_file_def="input.monomers.file_name",
     seq_file_def="input.sequence")
   self.working_phil = cmdline.work
   params = self.working_phil.extract()
   if len(params.input.pdb.file_name) == 0 :
     raise Sorry("At least one PDB file is required as input.")
   self.cif_file_names = params.input.monomers.file_name
   self.pdb_file_names = params.input.pdb.file_name
   # SYMMETRY HANDLING - PDB FILES
   self.crystal_symmetry = pdb_symm = None
   for pdb_file_name in params.input.pdb.file_name :
     pdb_symm = crystal_symmetry_from_any.extract_from(pdb_file_name)
     if (pdb_symm is not None) :
       break
   # DATA INPUT
   data_and_flags = hkl_symm = hkl_in = None
   if (params.input.xray_data.file_name is None) :
     if (require_data) :
       raise Sorry("At least one reflections file is required as input.")
   else :
     # FIXME this may still require that the data file has full crystal
     # symmetry defined (although for MTZ input this will not be a problem)
     make_sub_header("Processing X-ray data", out=self.log)
     hkl_in = file_reader.any_file(params.input.xray_data.file_name)
     hkl_in.check_file_type("hkl")
     hkl_server = hkl_in.file_server
     symm = hkl_server.miller_arrays[0].crystal_symmetry()
     if ((symm is None) or
         (symm.space_group() is None) or
         (symm.unit_cell() is None)) :
       if (pdb_symm is not None) :
         from iotbx.reflection_file_utils import reflection_file_server
         print >> self.log, \
           "No symmetry in X-ray data file - using PDB symmetry:"
         pdb_symm.show_summary(f=out, prefix="  ")
         hkl_server = reflection_file_server(
           crystal_symmetry=pdb_symm,
           reflection_files=[hkl_in.file_object])
       else :
         raise Sorry("No crystal symmetry information found in input files.")
     if (hkl_server is None) :
       hkl_server = hkl_in.file_server
     data_and_flags = mmtbx.utils.determine_data_and_flags(
       reflection_file_server=hkl_server,
       parameters=params.input.xray_data,
       data_parameter_scope="input.xray_data",
       flags_parameter_scope="input.xray_data.r_free_flags",
       prefer_anomalous=prefer_anomalous,
       force_non_anomalous=force_non_anomalous,
       log=self.log)
     self.intensity_flag = data_and_flags.intensity_flag
     self.raw_data = data_and_flags.raw_data
     self.raw_flags = data_and_flags.raw_flags
     self.test_flag_value = data_and_flags.test_flag_value
     self.f_obs = data_and_flags.f_obs
     self.r_free_flags = data_and_flags.r_free_flags
     self.miller_arrays = hkl_in.file_server.miller_arrays
     hkl_symm = self.raw_data.crystal_symmetry()
   if len(self.cif_file_names) > 0 :
     for file_name in self.cif_file_names :
       cif_obj = mmtbx.monomer_library.server.read_cif(file_name=file_name)
       self.cif_objects.append((file_name, cif_obj))
   # SYMMETRY HANDLING - COMBINED
   if (hkl_symm is not None) :
     use_symmetry = hkl_symm
   from iotbx.symmetry import combine_model_and_data_symmetry
   self.crystal_symmetry = combine_model_and_data_symmetry(
     model_symmetry=pdb_symm,
     data_symmetry=hkl_symm)
   if (self.crystal_symmetry is not None) and (self.f_obs is not None) :
     self.f_obs = self.f_obs.customized_copy(
       crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent().set_info(
         self.f_obs.info())
     self.r_free_flags = self.r_free_flags.customized_copy(
       crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent().set_info(
         self.r_free_flags.info())
   # EXPERIMENTAL PHASES
   target_name = "ml"
   if hasattr(params.input, "experimental_phases") :
     flag = params.input.use_experimental_phases
     if (flag in [True, Auto]) :
       phases_file = params.input.experimental_phases.file_name
       if (phases_file is None) :
         phases_file = params.input.xray_data.file_name
         phases_in = hkl_in
       else :
         phases_in = file_reader.any_file(phases_file)
         phases_in.check_file_type("hkl")
       phases_in.file_server.err = self.log # redirect error output
       space_group = self.crystal_symmetry.space_group()
       point_group = space_group.build_derived_point_group()
       hl_coeffs = mmtbx.utils.determine_experimental_phases(
         reflection_file_server = phases_in.file_server,
         parameters             = params.input.experimental_phases,
         log                    = self.log,
         parameter_scope        = "input.experimental_phases",
         working_point_group    = point_group,
         symmetry_safety_check  = True)
       if (hl_coeffs is not None) :
         hl_coeffs = hl_coeffs.map_to_asu()
         if hl_coeffs.anomalous_flag() :
           if (not self.f_obs.anomalous_flag()) :
             hl_coeffs = hl_coeffs.average_bijvoet_mates()
         elif self.f_obs.anomalous_flag() :
           hl_coeffs = hl_coeffs.generate_bijvoet_mates()
         self.hl_coeffs = hl_coeffs.matching_set(other=self.f_obs,
           data_substitute=(0,0,0,0))
         target_name = "mlhl"
   # PDB INPUT
   self.unknown_residues_flag = False
   self.unknown_residues_error_message = False
   if process_pdb_file :
     pdb_interp_params = getattr(params, "pdb_interpretation", None)
     if (pdb_interp_params is None) :
       pdb_interp_params = \
         mmtbx.monomer_library.pdb_interpretation.master_params.extract()
     make_sub_header("Processing PDB file(s)", out=self.log)
     pdb_combined = mmtbx.utils.combine_unique_pdb_files(
       file_names=params.input.pdb.file_name,)
     pdb_combined.report_non_unique(out=self.log)
     pdb_raw_records = pdb_combined.raw_records
     processed_pdb_files_srv = mmtbx.utils.process_pdb_file_srv(
       cif_objects=self.cif_objects,
       pdb_interpretation_params=pdb_interp_params,
       crystal_symmetry=self.crystal_symmetry,
       use_neutron_distances=params.input.scattering_table=="neutron",
       stop_for_unknowns=getattr(pdb_interp_params, "stop_for_unknowns",False),
       log=self.log)
     self.processed_pdb_file, self.pdb_inp = \
       processed_pdb_files_srv.process_pdb_files(
         raw_records = pdb_raw_records,
         stop_if_duplicate_labels = False,
         allow_missing_symmetry=\
           (self.crystal_symmetry is None) and (not require_data))
     error_msg = self.processed_pdb_file.all_chain_proxies.\
       fatal_problems_message(
         ignore_unknown_scattering_types=False,
         ignore_unknown_nonbonded_energy_types=False)
     if (error_msg is not None) :
       self.unknown_residues_flag = True
       self.unknown_residues_error_message = error_msg
     self.geometry = self.processed_pdb_file.geometry_restraints_manager(
       show_energies=False)
     assert (self.geometry is not None)
     self.xray_structure = self.processed_pdb_file.xray_structure()
     chain_proxies = self.processed_pdb_file.all_chain_proxies
     self.pdb_hierarchy = chain_proxies.pdb_hierarchy
   else :
     pdb_file_object = mmtbx.utils.pdb_file(
       pdb_file_names=params.input.pdb.file_name,
       cif_objects=self.cif_objects,
       crystal_symmetry=self.crystal_symmetry,
       log=self.log)
     self.pdb_inp = pdb_file_object.pdb_inp
     self.pdb_hierarchy = self.pdb_inp.construct_hierarchy()
     if (remove_unknown_scatterers) :
       known_sel = self.pdb_hierarchy.atom_selection_cache().selection(
         "not element X")
       if (known_sel.count(True) != len(known_sel)) :
         self.pdb_hierarchy = self.pdb_hierarchy.select(known_sel)
         self.xray_structure = self.pdb_hierarchy.extract_xray_structure(
           crystal_symmetry=self.crystal_symmetry)
     self.pdb_hierarchy.atoms().reset_i_seq()
     if (self.xray_structure is None) :
       self.xray_structure = self.pdb_inp.xray_structure_simple(
         crystal_symmetry=self.crystal_symmetry)
   # wavelength
   if (params.input.energy is not None) :
     if (params.input.wavelength is not None) :
       raise Sorry("Both wavelength and energy have been specified!")
     params.input.wavelength = 12398.424468024265 / params.input.energy
   if (set_wavelength_from_model_header and params.input.wavelength is None) :
     wavelength = self.pdb_inp.extract_wavelength()
     if (wavelength is not None) :
       print >> self.log, ""
       print >> self.log, "Using wavelength = %g from PDB header" % wavelength
       params.input.wavelength = wavelength
   # set scattering table
   if (data_and_flags is not None) :
     self.xray_structure.scattering_type_registry(
       d_min=self.f_obs.d_min(),
       table=params.input.scattering_table)
     if ((params.input.wavelength is not None) and
         (set_inelastic_form_factors is not None)) :
       self.xray_structure.set_inelastic_form_factors(
         photon=params.input.wavelength,
         table=set_inelastic_form_factors)
     make_sub_header("xray_structure summary", out=self.log)
     self.xray_structure.scattering_type_registry().show(out = self.log)
     self.xray_structure.show_summary(f=self.log)
   # FMODEL SETUP
   if (create_fmodel) and (data_and_flags is not None) :
     make_sub_header("F(model) initialization", out=self.log)
     skip_twin_detection = getattr(params.input, "skip_twin_detection", None)
     twin_law = getattr(params.input, "twin_law", None)
     if (twin_law is Auto) :
       if (self.hl_coeffs is not None) :
         raise Sorry("Automatic twin law determination not supported when "+
           "experimental phases are used.")
     elif (skip_twin_detection is not None) :
       twin_law = Auto
     if (twin_law is Auto) :
       print >> self.log, "Twinning will be detected automatically."
       self.fmodel = mmtbx.utils.fmodel_simple(
         xray_structures=[self.xray_structure],
         scattering_table=params.input.scattering_table,
         f_obs=self.f_obs,
         r_free_flags=self.r_free_flags,
         skip_twin_detection=skip_twin_detection,
         target_name=target_name,
         log=self.log)
     else :
       if ((twin_law is not None) and (self.hl_coeffs is not None)) :
         raise Sorry("Automatic twin law determination not supported when "+
           "experimental phases are used.")
       self.fmodel = mmtbx.utils.fmodel_manager(
         f_obs=self.f_obs,
         xray_structure=self.xray_structure,
         r_free_flags=self.r_free_flags,
         twin_law=params.input.twin_law,
         hl_coeff=self.hl_coeffs,
         target_name=target_name)
       self.fmodel.update_all_scales(
         params=None,
         log=self.log,
         optimize_mask=True,
         show=True)
     self.fmodel.info().show_rfactors_targets_scales_overall(out=self.log)
   # SEQUENCE
   if (params.input.sequence is not None) :
     seq_file = file_reader.any_file(params.input.sequence,
       force_type="seq",
       raise_sorry_if_errors=True)
     self.sequence = seq_file.file_object
   # UNMERGED DATA
   self.unmerged_i_obs = None
   if hasattr(params.input, "unmerged_data") :
     if (params.input.unmerged_data.file_name is not None) :
       self.unmerged_i_obs = load_and_validate_unmerged_data(
         f_obs=self.f_obs,
         file_name=params.input.unmerged_data.file_name,
         data_labels=params.input.unmerged_data.labels,
         log=self.log)
   self.params = params
   print >> self.log, ""
   print >> self.log, "End of input processing"
def simul_utils(args):
  if len(args)==0:
    print_help()
  elif ( "--help" in args ):
    print_help()
  elif ( "--h" in args ):
    print_help()
  elif ("-h" in args ):
    print_help()
  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="map_coefs")

    print >> log, "#phil __OFF__"
    print >> log, "======================"
    print >> log, "          SIMUL       "
    print >> log, "A data simulation tool"
    print >> log, "======================"
    print >> log


    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()
    """
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    """
    # now get the unit cell from the pdb file

    hkl_xs = crystal_symmetry_from_any.extract_from(
      file_name=params.simul_utils.input.xray_data.file_name)
    pdb_xs = crystal_symmetry_from_any.extract_from(
      file_name=params.simul_utils.input.model.file_name)

    phil_xs = crystal.symmetry(
      unit_cell=params.simul_utils.input.unit_cell,
      space_group_info=params.simul_utils.input.space_group  )


    combined_xs = select_crystal_symmetry(
      None,phil_xs, [pdb_xs],[hkl_xs])

    # inject the unit cell and symmetry in the phil scope please
    params.simul_utils.input.unit_cell = combined_xs.unit_cell()
    params.simul_utils.input.space_group = \
      sgtbx.space_group_info( group = combined_xs.space_group() )

    print >> log, "#phil __ON__"
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    print >> log, "#phil __END__"

    if params.simul_utils.input.unit_cell is None:
      raise Sorry("unit cell not specified")
    if params.simul_utils.input.space_group is None:
      raise Sorry("space group not specified")
    if params.simul_utils.input.xray_data.file_name is None:
      raise Sorry("Xray data not specified")
    if params.simul_utils.input.model.file_name is None:
      raise Sorry("pdb file with  model not specified")

    #-----------------------------------------------------------
    #
    # step 1: read in the reflection file
    #
    phil_xs = crystal.symmetry(
      unit_cell=params.simul_utils.input.unit_cell,
      space_group_info=params.simul_utils.input.space_group  )

    xray_data_server =  reflection_file_utils.reflection_file_server(
      crystal_symmetry = phil_xs,
      force_symmetry = True,
      reflection_files=[])

    miller_array = None

    miller_array = xray_data_server.get_xray_data(
      file_name = params.simul_utils.input.xray_data.file_name,
      labels = params.simul_utils.input.xray_data.labels,
      ignore_all_zeros = True,
      parameter_scope = 'simul_utils.input.xray_data',
      parameter_name = 'labels'
      )

    info = miller_array.info()

    miller_array = miller_array.map_to_asu()

    miller_array = miller_array.select(
      miller_array.indices() != (0,0,0))

    miller_array = miller_array.select(
      miller_array.data() > 0 )
    if  miller_array.sigmas() is not None:
      miller_array = miller_array.select(
        miller_array.sigmas() > 0 )

    if (miller_array.is_xray_intensity_array()):
      miller_array = miller_array.f_sq_as_f()
    elif (miller_array.is_complex_array()):
      miller_array = abs(miller_array)

    miller_array.set_info(info)
    print >> log
    print >> log, "Summary info of observed data"
    print >> log, "============================="
    miller_array.show_summary(f=log)
    print >> log


    free_flags = miller_array.generate_r_free_flags()



    #--------------------------------------------------------------------
    # Step 2: get an xray structure from the PDB file
    #
    model = pdb.input(file_name=params.simul_utils.input.model.file_name).xray_structure_simple(
                      crystal_symmetry=phil_xs,
                      )
    print >> log, "Atomic model summary"
    print >> log, "===================="
    model.show_summary()
    print >> log

    #-------------------------------------------------------------------
    # Step 3: make an F_model object to get model phases and amplitudes
    #
    print >> log, "Performing bulk solvent scaling"
    print >> log, "==============================="
    print >> log
    print >> log

    f_model_object = f_model.manager(
        f_obs = miller_array,
        r_free_flags = free_flags,
        xray_structure = model )
    f_model_object.update_solvent_and_scale(out=log)
    fmodel = abs( f_model_object.f_model() ).set_observation_type( miller_array )

    mockfmodel = None
    if params.simul_utils.input.mock_model.file_name is not None:
      print >> log, "Reading in mock model"
      print >> log, "====================="
      print >> log
      print >> log
      mock_model = pdb.input(file_name=params.simul_utils.input.mock_model.file_name).xray_structure_simple(
                             crystal_symmetry=phil_xs)
      mock_f_model = f_model.manager(
        f_obs = miller_array,
        r_free_flags = free_flags,
        xray_structure = mock_model )

      mock_f_model.update(
        k_sol  = f_model_object.k_sol() ,
        b_sol  = f_model_object.b_sol() ,
        b_cart = f_model_object.b_cart()
      )
      mockfmodel = abs( mock_f_model.f_model() ).set_observation_type( miller_array )
    else:
      mockfmodel = fmodel.deep_copy()




    print >> log, "Making new data"
    print >> log, "==============="
    print >> log
    print >> log

    new_data_builder = error_swap( miller_array,
                                   fmodel,
                                   mockfmodel )
    new_data = new_data_builder.new_obs
    # we now have to write the data actually

    print >> log, "Writing new data set"
    print >> log, "===================="

    mtz_dataset = new_data.as_mtz_dataset(
      column_root_label="FOBS")
    mtz_dataset.mtz_object().write(
      file_name=params.simul_utils.output.hklout)
Exemple #42
0
def run(args):

    if len(args) == 0:
        master_params.show(expert_level=0)
    elif ("--help" in args):
        print("no help available")
    elif ("--h" in args):
        print("no help available")
    elif ("--show_defaults" in args):
        master_params.show(expert_level=0)
    elif ("--show_defaults_all" in args):
        master_params.show(expert_level=10)

    else:
        log = multi_out()
        if (not "--quiet" in args):
            log.register(label="stdout", file_object=sys.stdout)
        string_buffer = StringIO()
        string_buffer_plots = StringIO()
        log.register(label="log_buffer", file_object=string_buffer)

        log_plots = StringIO()
        print("#phil __OFF__", file=log)
        print(file=log)
        print(date_and_time(), file=log)
        print(file=log)
        print(file=log)

        phil_objects = []
        argument_interpreter = master_params.command_line_argument_interpreter(
            home_scope="scaling")

        reflection_file = None

        for arg in args:
            command_line_params = None
            arg_is_processed = False
            if arg == '--quiet':
                arg_is_processed = True
                ## The associated action with this keyword is implemented above
            if (os.path.isfile(arg)):  ## is this a file name?
                ## Check if this is a phil file
                try:
                    command_line_params = iotbx.phil.parse(file_name=arg)
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass
                if command_line_params is not None:
                    phil_objects.append(command_line_params)
                    arg_is_processed = True
                ## Check if this file is a reflection file
                if command_line_params is None:
                    reflection_file = reflection_file_reader.any_reflection_file(
                        file_name=arg, ensure_read_access=False)
                if (reflection_file is not None):
                    reflection_file = arg
                    arg_is_processed = True
            ## If it is not a file, it must be a phil command
            else:
                try:
                    command_line_params = argument_interpreter.process(arg=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass

            if not arg_is_processed:
                print("##----------------------------------------------##",
                      file=log)
                print("## Unknown phil-file or phil-command:", arg, file=log)
                print("##----------------------------------------------##",
                      file=log)
                print(file=log)
                raise Sorry("Unknown file format or phil command: %s" % arg)

        effective_params = master_params.fetch(sources=phil_objects)
        params = effective_params.extract()

        ## Now please read in the reflections files

        ## get symmetry and cell data first please
        ## By default, the native cell and symmetry are used
        ## as reference
        crystal_symmetry_nat = None
        crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
            file_name=params.scaling.input.xray_data.native.file_name)

        if params.scaling.input.xray_data.space_group is None:
            params.scaling.input.xray_data.space_group =\
              crystal_symmetry_nat.space_group_info()
            print("Using symmetry of native data", file=log)

        if params.scaling.input.xray_data.unit_cell is None:
            params.scaling.input.xray_data.unit_cell =\
              crystal_symmetry_nat.unit_cell()
            print("Using cell of native data", file=log)

        ## Check if a unit cell is defined
        if params.scaling.input.xray_data.space_group is None:
            raise Sorry("No space group defined")
        if params.scaling.input.xray_data.unit_cell is None:
            raise Sorry("No unit cell defined")

        crystal_symmetry = crystal_symmetry = crystal.symmetry(
            unit_cell=params.scaling.input.xray_data.unit_cell,
            space_group_symbol=str(params.scaling.input.xray_data.space_group))

        effective_params = master_params.fetch(sources=phil_objects)
        new_params = master_params.format(python_object=params)
        print("Effective parameters", file=log)
        print("#phil __ON__", file=log)
        new_params.show(out=log,
                        expert_level=params.scaling.input.expert_level)
        print("#phil __END__", file=log)
        print(file=log)

        ## define a xray data server
        xray_data_server = reflection_file_utils.reflection_file_server(
            crystal_symmetry=crystal_symmetry,
            force_symmetry=True,
            reflection_files=[])

        ## Read in native data and make appropriatre selections
        miller_array_native = None
        miller_array_native = xray_data_server.get_xray_data(
            file_name=params.scaling.input.xray_data.native.file_name,
            labels=params.scaling.input.xray_data.native.labels,
            ignore_all_zeros=True,
            parameter_scope='scaling.input.SIR_scale.xray_data.native')
        info_native = miller_array_native.info()
        miller_array_native = miller_array_native.map_to_asu().select(
            miller_array_native.indices() != (0, 0, 0))
        miller_array_native = miller_array_native.select(
            miller_array_native.data() > 0)
        ## Convert to amplitudes
        if (miller_array_native.is_xray_intensity_array()):
            miller_array_native = miller_array_native.f_sq_as_f()
        elif (miller_array_native.is_complex_array()):
            miller_array_native = abs(miller_array_native)
        if not miller_array_native.is_real_array():
            raise Sorry("miller_array_native is not a real array")
        miller_array_native.set_info(info=info_native)

        ## Read in derivative data and make appropriate selections
        miller_array_derivative = None
        miller_array_derivative = xray_data_server.get_xray_data(
            file_name=params.scaling.input.xray_data.derivative.file_name,
            labels=params.scaling.input.xray_data.derivative.labels,
            ignore_all_zeros=True,
            parameter_scope='scaling.input.SIR_scale.xray_data.derivative')
        info_derivative = miller_array_derivative.info()
        miller_array_derivative = miller_array_derivative.map_to_asu().select(
            miller_array_derivative.indices() != (0, 0, 0))
        miller_array_derivative = miller_array_derivative.select(
            miller_array_derivative.data() > 0)
        ## Convert to amplitudes
        if (miller_array_derivative.is_xray_intensity_array()):
            miller_array_derivative = miller_array_derivative.f_sq_as_f()
        elif (miller_array_derivative.is_complex_array()):
            miller_array_derivative = abs(miller_array_derivative)
        if not miller_array_derivative.is_real_array():
            raise Sorry("miller_array_derivative is not a real array")
        miller_array_derivative.set_info(info=info_derivative)

        ## As this is a SIR case, we will remove any anomalous pairs
        if miller_array_derivative.anomalous_flag():
            miller_array_derivative = miller_array_derivative.average_bijvoet_mates()\
            .set_observation_type( miller_array_derivative )
        if miller_array_native.anomalous_flag():
            miller_array_native = miller_array_native.average_bijvoet_mates()\
            .set_observation_type( miller_array_native )

        ## Print info
        print(file=log)
        print("Native data", file=log)
        print("===========", file=log)
        miller_array_native.show_comprehensive_summary(f=log)
        print(file=log)
        native_pre_scale = pre_scale.pre_scaler(
            miller_array_native,
            params.scaling.input.scaling_strategy.pre_scaler_protocol,
            params.scaling.input.basic)
        miller_array_native = native_pre_scale.x1.deep_copy()
        del native_pre_scale

        print(file=log)
        print("Derivative data", file=log)
        print("===============", file=log)
        miller_array_derivative.show_comprehensive_summary(f=log)
        print(file=log)
        derivative_pre_scale = pre_scale.pre_scaler(
            miller_array_derivative,
            params.scaling.input.scaling_strategy.pre_scaler_protocol,
            params.scaling.input.basic)
        miller_array_derivative = derivative_pre_scale.x1.deep_copy()
        del derivative_pre_scale

        scaler = fa_estimation.combined_scaling(
            miller_array_native, miller_array_derivative,
            params.scaling.input.scaling_strategy.iso_protocol)

        miller_array_native = scaler.x1.deep_copy()
        miller_array_derivative = scaler.x2.deep_copy()
        del scaler

        print(file=log)
        print("Making delta f's", file=log)
        print("----------------", file=log)
        print(file=log)

        delta_gen = pair_analyses.delta_generator(miller_array_native,
                                                  miller_array_derivative)
        print(file=log)
        print("writing mtz file", file=log)
        print("----------------", file=log)
        print(file=log)

        ## some assertions to make sure nothing went weerd
        assert miller_array_native.observation_type() is not None
        assert miller_array_derivative.observation_type() is not None
        assert delta_gen.abs_delta_f.observation_type() is not None

        ## Please write out the abs_delta_f array

        mtz_dataset = delta_gen.abs_delta_f.as_mtz_dataset(
            column_root_label='F' + params.scaling.input.output.outlabel)
        mtz_dataset.mtz_object().write(
            file_name=params.scaling.input.output.hklout)
Exemple #43
0
def cmd_run(args, validated=False, out=sys.stdout):
  if (len(args) == 0):
    print >> out, "-"*79
    print >> out, "                               phenix.polder"
    print >> out, "-"*79
    print >> out, legend
    print >> out, "-"*79
    master_params.show(out=out)
    return
  log = multi_out()
  log.register("stdout", out)
  log_file_name = "polder.log"
  logfile = open(log_file_name, "w")
  log.register("logfile", logfile)
  print >> log, "phenix.polder is running..."
  print >> log, "input parameters:\n", args
  parsed = master_params
  inputs = mmtbx.utils.process_command_line_args(args = args,
    master_params = parsed)
  #inputs.params.show() #check
  params = inputs.params.extract()
  # check model file
  if len(inputs.pdb_file_names) == 0:
    if (params.model_file_name is None):
      raise Sorry("No model file found.")
  elif (len(inputs.pdb_file_names) == 1):
    params.model_file_name = inputs.pdb_file_names[0]
  else:
    raise Sorry("Only one model file should be given")
  # check reflection file
  reflection_files = inputs.reflection_files
  if (len(reflection_files) == 0):
    if (params.reflection_file_name is None):
      raise Sorry("No reflection file found.")
    else:
      hkl_in = file_reader.any_file(params.reflection_file_name,
        force_type="hkl")
      hkl_in.assert_file_type("hkl")
      reflection_files = [ hkl_in.file_object ]
  # crystal symmetry
  crystal_symmetry = None
  crystal_symmetry = inputs.crystal_symmetry
  if (crystal_symmetry is None):
    crystal_symmetries = []
    for f in [str(params.model_file_name), str(params.reflection_file_name)]:
      cs = crystal_symmetry_from_any.extract_from(f)
      if(cs is not None): crystal_symmetries.append(cs)
    if(len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0]
    elif(len(crystal_symmetries) == 0):
      raise Sorry("No crystal symmetry found.")
    else:
      if(not crystal_symmetries[0].is_similar_symmetry(crystal_symmetries[1])):
        raise Sorry("Crystal symmetry mismatch between different files.")
      crystal_symmetry = crystal_symmetries[0]
  f_obs, r_free_flags = None, None
  rfs = reflection_file_utils.reflection_file_server(
    crystal_symmetry = crystal_symmetry,
    force_symmetry   = True,
    reflection_files = reflection_files,
    err              = StringIO())
  parameters = mmtbx.utils.data_and_flags_master_params().extract()
  if (params.data_labels is not None):
    parameters.labels = params.data_labels
  if (params.r_free_flags_labels is not None):
    parameters.r_free_flags.label = params.r_free_flags_labels
  determined_data_and_flags = mmtbx.utils.determine_data_and_flags(
    reflection_file_server = rfs,
    parameters             = parameters,
    keep_going             = True,
    log                    = StringIO())
  f_obs = determined_data_and_flags.f_obs
  if (params.data_labels is None):
    params.data_labels = f_obs.info().label_string()
  if (params.reflection_file_name is None):
    params.reflection_file_name = parameters.file_name
  r_free_flags = determined_data_and_flags.r_free_flags
  assert f_obs is not None
  print >> log,  "Input data:"
  print >> log, "  Iobs or Fobs:", f_obs.info().labels
  if (r_free_flags is not None):
    print >> log, "  Free-R flags:", r_free_flags.info().labels
    params.r_free_flags_labels = r_free_flags.info().label_string()
  else:
    print >> log, "  Free-R flags: Not present"
  model_basename = os.path.basename(params.model_file_name.split(".")[0])
  if (len(model_basename) > 0 and
    params.output_file_name_prefix is None):
    params.output_file_name_prefix = model_basename
  print params.output_file_name_prefix
  new_params =  master_params.format(python_object=params)
  new_params.show()
  if (not validated):
    validate_params(params)
  pdb_input = iotbx.pdb.input(file_name = params.model_file_name)
  pdb_hierarchy = pdb_input.construct_hierarchy()
  xray_structure = pdb_hierarchy.extract_xray_structure(
    crystal_symmetry = crystal_symmetry)
  # DON'T USE:
  # xray_structure = pdb_input.xray_structure_simple()
  # atom order might be wrong
  mmtbx.utils.setup_scattering_dictionaries(
    scattering_table = params.scattering_table,
    xray_structure   = xray_structure,
    d_min            = f_obs.d_min())
  #if f_obs is not None:
  f_obs = f_obs.resolution_filter(
    d_min = params.high_resolution,
    d_max = params.low_resolution)
  if (r_free_flags is not None):
    r_free_flags = r_free_flags.resolution_filter(
      d_min = params.high_resolution,
      d_max = params.low_resolution)
# Grab case that data are anomalous
  if (f_obs.anomalous_flag()):
    f_obs, r_free_flags = prepare_f_obs_and_flags(
      f_obs        = f_obs,
      r_free_flags = r_free_flags)
  cpm_obj = compute_polder_map(
    f_obs          = f_obs,
    r_free_flags   = r_free_flags,
    xray_structure = xray_structure,
    pdb_hierarchy  = pdb_hierarchy,
    params         = params,
    log            = log)
# Significance check
  fmodel = mmtbx.f_model.manager(
    f_obs          = f_obs,
    r_free_flags   = r_free_flags,
    xray_structure = xray_structure)
  fmodel.update_all_scales(remove_outliers=False, fast=True)
  f_obs_1 = abs(fmodel.f_model())
  fmodel.update_xray_structure(xray_structure=cpm_obj.xray_structure_noligand,
    update_f_calc=True, update_f_mask=True, force_update_f_mask=True)
  # PVA: do we need it? fmodel.update_all_scales(remove_outliers=False)
  f_obs_2 = abs(fmodel.f_model())
  xrs_selected = cpm_obj.pdb_hierarchy_selected.extract_xray_structure(
    crystal_symmetry = f_obs.crystal_symmetry())
  f_calc = f_obs.structure_factors_from_scatterers(
    xray_structure = cpm_obj.xray_structure_noligand).f_calc()
  f_mask = f_obs.structure_factors_from_map(
    map            = cpm_obj.mask_polder,
    use_scale      = True,
    anomalous_flag = False,
    use_sg         = False)
  def get_poler_diff_map(f_obs):
    fmodel = mmtbx.f_model.manager(
      f_obs        = f_obs,
      r_free_flags = r_free_flags,
      f_calc       = f_calc,
      f_mask       = f_mask)
    fmodel.update_all_scales(remove_outliers=False)
    mc_diff = map_tools.electron_density_map(
      fmodel = fmodel).map_coefficients(
        map_type         = "mFo-DFc",
        isotropize       = True,
        fill_missing     = False)
    fft_map = miller.fft_map(
      crystal_gridding     = cpm_obj.crystal_gridding,
      fourier_coefficients = mc_diff)
    fft_map.apply_sigma_scaling()
    map_data = fft_map.real_map_unpadded()
    return mmtbx.utils.extract_box_around_model_and_map(
      xray_structure = xrs_selected,
      map_data       = map_data,
      box_cushion    = 2.1)
  box_1=get_poler_diff_map(f_obs = f_obs_1)
  box_2=get_poler_diff_map(f_obs = f_obs_2)
  box_3=get_poler_diff_map(f_obs = f_obs)
  sites_cart_box = box_1.xray_structure_box.sites_cart()
  sel = maptbx.grid_indices_around_sites(
    unit_cell  = box_1.xray_structure_box.unit_cell(),
    fft_n_real = box_1.map_box.focus(),
    fft_m_real = box_1.map_box.all(),
    sites_cart = sites_cart_box,
    site_radii = flex.double(sites_cart_box.size(), 2.0))
  b1 = box_1.map_box.select(sel).as_1d()
  b2 = box_2.map_box.select(sel).as_1d()
  b3 = box_3.map_box.select(sel).as_1d()
  print >> log, "Map 1: calculated Fobs with ligand"
  print >> log, "Map 2: calculated Fobs without ligand"
  print >> log, "Map 3: real Fobs data"
  print >>log, "CC(1,2): %6.4f"%flex.linear_correlation(x=b1,y=b2).coefficient()
  print >>log, "CC(1,3): %6.4f"%flex.linear_correlation(x=b1,y=b3).coefficient()
  print >>log, "CC(2,3): %6.4f"%flex.linear_correlation(x=b2,y=b3).coefficient()
  ### D-function
  b1 = maptbx.volume_scale_1d(map=b1, n_bins=10000).map_data()
  b2 = maptbx.volume_scale_1d(map=b2, n_bins=10000).map_data()
  b3 = maptbx.volume_scale_1d(map=b3, n_bins=10000).map_data()
  print >> log, "Peak CC:"
  print >>log, "CC(1,2): %6.4f"%flex.linear_correlation(x=b1,y=b2).coefficient()
  print >>log, "CC(1,3): %6.4f"%flex.linear_correlation(x=b1,y=b3).coefficient()
  print >>log, "CC(2,3): %6.4f"%flex.linear_correlation(x=b2,y=b3).coefficient()
  cutoffs = flex.double(
    [i/10. for i in range(1,10)]+[i/100 for i in range(91,100)])
  d12 = maptbx.discrepancy_function(map_1=b1, map_2=b2, cutoffs=cutoffs)
  d13 = maptbx.discrepancy_function(map_1=b1, map_2=b3, cutoffs=cutoffs)
  d23 = maptbx.discrepancy_function(map_1=b2, map_2=b3, cutoffs=cutoffs)
  print >> log, "q    D(1,2) D(1,3) D(2,3)"
  for c,d12_,d13_,d23_ in zip(cutoffs,d12,d13,d23):
    print >> log, "%4.2f %6.4f %6.4f %6.4f"%(c,d12_,d13_,d23_)
  ###
  if(params.debug):
    box_1.write_ccp4_map(file_name="box_1_polder.ccp4")
    box_2.write_ccp4_map(file_name="box_2_polder.ccp4")
    box_3.write_ccp4_map(file_name="box_3_polder.ccp4")
    cpm_obj.pdb_hierarchy_selected.adopt_xray_structure(
      box_1.xray_structure_box)
    cpm_obj.pdb_hierarchy_selected.write_pdb_file(file_name="box_polder.pdb",
      crystal_symmetry=box_1.box_crystal_symmetry)
  #
  print >> log, "Finished."
  return True
Exemple #44
0
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    for j, se in enumerate(zip(starts, ends)):
        s, e = se
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    for s, e in zip(starts, ends):
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(uc, indices, intensities, sigmas))

    from collections import defaultdict

    hist = defaultdict(int)

    fout = open('common_oh.dat', 'w')

    for j, f in enumerate(frames):
        hp = f.hand_pairs()
        fout.write('%4d %d\n' % (j, hp))
        hist[int(hp)] += 1

    fout.close()

    for b in sorted(hist):
        print b, hist[b]


    

    return
def run(args):
  phil = iotbx.phil.process_command_line(args=args, master_string=master_phil).show()
  work_params = phil.work.extract()
  log = open("%s_%s_merging.log" % (work_params.output.prefix,work_params.scaling.algorithm), "w")
  out = multi_out()
  out.register("log", log, atexit_send_to=None)
  out.register("stdout", sys.stdout)

  print >> out, "Target unit cell and space group:"
  print >> out, "  ", work_params.target_unit_cell
  print >> out, "  ", work_params.target_space_group

  miller_set = symmetry(
      unit_cell=work_params.target_unit_cell,
      space_group_info=work_params.target_space_group
    ).build_miller_set(
      anomalous_flag=not work_params.merge_anomalous,
      d_min=work_params.d_min)
  from xfel.merging.general_fcalc import random_structure
  i_model = random_structure(work_params)

# ---- Augment this code with any special procedures for x scaling
  scaler = xscaling_manager(
    miller_set=miller_set,
    i_model=i_model,
    params=work_params,
    log=out)
  scaler.read_all_mysql()
  print "finished reading the database"
  sg = miller_set.space_group()

  hkl_asu = scaler.observations["hkl_id"]
  imageno = scaler.observations["frame_id"]
  intensi = scaler.observations["i"]
  lookup = scaler.millers["merged_asu_hkl"]
  origH = scaler.observations["H"]
  origK = scaler.observations["K"]
  origL = scaler.observations["L"]

  from cctbx.array_family import flex

  print "# observations from the database",len(scaler.observations["hkl_id"])
  hkl = flex.miller_index(flex.select(lookup,hkl_asu))
  from cctbx import miller

  hkl_list = miller_set.customized_copy(indices = hkl)

  ARRAY = miller.array(miller_set = hkl_list, data = intensi)
  LATTICES = miller.array(miller_set = hkl_list, data = imageno)

  from cctbx.merging.brehm_diederichs import run_multiprocess, run

  L = (ARRAY, LATTICES) # tuple(data,lattice_id)
  from libtbx import easy_pickle
  presort_file = work_params.output.prefix+"_intensities_presort.pickle"
  print "pickling these intensities to", presort_file
  easy_pickle.dump(presort_file,L)

    ######  INPUTS #######
    #       data = miller array: ASU miller index + intensity (sigmas not implemented yet)
    #       lattice_id = flex double: assignment of each miller index to a lattice number
    ######################
  if work_params.nproc < 5:
    print "Sorting the lattices with 1 processor"
    result = run(L,nproc=1,verbose=True)
  else:
    print "Sorting the lattices with %d processors"%work_params.nproc
    result = run_multiprocess(L,nproc=work_params.nproc, verbose=False)
  for key in result.keys():
    print key,len(result[key])

  # 2) pickle the postsort (reindexed) ARRAY, LATTICES XXX not done yet; not clear if needed

  reverse_lookup = {}
  frame_id_list = list(scaler.frames_mysql["frame_id"])
  for key in result.keys():
    for frame in result[key]:
      frame_idx = frame_id_list.index(frame)
      reverse_lookup[scaler.frames_mysql["unique_file_name"][frame_idx]] = key

  lookup_file = work_params.output.prefix+"_lookup.pickle"
  reverse_lookup_file = work_params.output.prefix+"_reverse_lookup.pickle"
  easy_pickle.dump(lookup_file, result)
  easy_pickle.dump(reverse_lookup_file, reverse_lookup)
Exemple #46
0
def reindex_utils(args):
  if len(args)==0:
    print_help()
  elif ( "--help" in args ):
    print_help()
  elif ( "--h" in args ):
    print_help()
  elif ("-h" in args ):
    print_help()
  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="map_coefs")

    print >> log, "#phil __OFF__"
    print >> log, "================="
    print >> log, "    REINDEX      "
    print >> log, "A reindexing tool"
    print >> log, "================="
    print >> log


    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = master_params.fetch(sources=phil_objects)
    params_root = effective_params.extract()
    params = params_root.reindex_utils

    # now get the unit cell from the files
    hkl_xs = None
    pdb_xs = None
    if params.input.xray_data.file_name is not None:
      hkl_xs = crystal_symmetry_from_any.extract_from(
        file_name=params.input.xray_data.file_name)
    if params.input.model.file_name is not None:
      pdb_xs = crystal_symmetry_from_any.extract_from(
        file_name=params.input.model.file_name)

    phil_xs = crystal.symmetry(
      unit_cell=params.input.unit_cell,
      space_group_info=params.input.space_group  )


    combined_xs = crystal.select_crystal_symmetry(
      None,phil_xs, [pdb_xs],[hkl_xs])

    # inject the unit cell and symmetry in the phil scope please
    params.input.unit_cell = combined_xs.unit_cell()
    params.input.space_group = combined_xs.space_group_info()

    print >> log, "#phil __ON__"
    new_params =  master_params.format(python_object=params_root)
    new_params.show(out=log)
    print >> log, "#phil __END__"

    if params.input.unit_cell is None:
      raise Sorry("unit cell not specified")
    if params.input.space_group is None:
      raise Sorry("space group not specified")

    #-----------------------------------------------------------
    #
    # step 1: read in the reflection file
    #
    miller_array = None
    if  params.input.xray_data.file_name is not None:
      phil_xs = crystal.symmetry(
        unit_cell=params.input.unit_cell,
        space_group_info=params.input.space_group  )

      xray_data_server =  reflection_file_utils.reflection_file_server(
        crystal_symmetry = phil_xs,
        force_symmetry = True,
        reflection_files=[])

      miller_array = xray_data_server.get_xray_data(
        file_name = params.input.xray_data.file_name,
        labels = params.input.xray_data.labels,
        ignore_all_zeros = True,
        parameter_scope = 'reindex_utils.input.xray_data',
        parameter_name = 'labels'
        )

      info = miller_array.info()

      miller_array = miller_array.map_to_asu()

      miller_array = miller_array.select(
        miller_array.indices() != (0,0,0))

      miller_array = miller_array.select(
        miller_array.data() > 0 )
      if  miller_array.sigmas() is not None:
        miller_array = miller_array.select(
          miller_array.sigmas() > 0 )

      if (miller_array.is_xray_intensity_array()):
        miller_array = miller_array.f_sq_as_f()
      elif (miller_array.is_complex_array()):
        miller_array = abs(miller_array)

      miller_array.set_info(info)
      print >> log
      print >> log, "Summary info of observed data"
      print >> log, "============================="
      miller_array.show_summary(f=log)
      print >> log


    #----------------------------------------------------------------
    # Step 2: get an xray structure from the PDB file
    #
    pdb_model = None

    if params.input.model.file_name is not None:
      pdb_model = iotbx.pdb.input(
        file_name=params.input.model.file_name)
      model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
      print >> log, "Atomic model summary"
      print >> log, "===================="
      model.show_summary()
      print >> log

    if params.parameters.action=="reindex":
      #----------------------------------------------------------------
      # step 3: get the reindex laws
      to_niggli    = phil_xs.change_of_basis_op_to_niggli_cell()
      to_reference = phil_xs.change_of_basis_op_to_reference_setting()
      to_inverse   = phil_xs.change_of_basis_op_to_inverse_hand()
      cb_op = None
      pr = params.parameters.reindex
      if (pr.standard_laws == "niggli"):
        cb_op = to_niggli
      elif (pr.standard_laws == "reference_setting"):
        cb_op = to_reference
      elif (pr.standard_laws == "invert"):
        cb_op = to_inverse
      else:
        cb_op = sgtbx.change_of_basis_op(pr.user_supplied_law)

      if cb_op is None:
        raise Sorry("No change of basis operation is supplied.")
      if params.parameters.inverse:
        cb_op = cb_op.inverse()

      print >> log, "Supplied reindexing law:"
      print >> log, "========================"
      print >> log, "hkl notation: ", cb_op.as_hkl()
      print >> log, "xyz notation: ", cb_op.as_xyz()
      print >> log, "abc notation: ", cb_op.as_abc()
      #----------------------------------------------------------------
      # step 4: do the reindexing
      #
      # step 4a: first do the miller array object
      new_miller_array = None
      if miller_array is not None:
        new_miller_array = miller_array.change_basis( cb_op )
      #
      # step 4b: the xray structure
      new_model = None
      if pdb_model is not None:
        new_model = model.change_basis( cb_op )

      #----------------------------------------------------------------
      # step 5a: write the new mtz file
      print >> log
      print >> log, "The data and model have been reindexed"
      print >> log, "--------------------------------------"
      print >> log
      print >> log, "Writing output files...."
      if miller_array is not None:
        print >> log, "writing mtz file with name %s" % (params.output.hklout)
        mtz_dataset = new_miller_array.as_mtz_dataset(
          column_root_label="FOBS")
        mtz_dataset.mtz_object().write(file_name=params.output.hklout)

      #step 5b: write the new pdb file
      if new_model is not None:
        pdb_file = open(params.output.xyzout, 'w')
        print >> log, "Wring pdb file to: %s" % params.output.xyzout
        write_as_pdb_file(
          input_pdb = pdb_model,
          input_xray_structure = new_model,
          out = pdb_file,
          chain_id_increment = params.parameters.chain_id_increment,
          additional_remark = "Generated by mmtbx reindex")

        print >> pdb_file, "END"
        pdb_file.close()
      if ( [miller_array,new_model]).count(None)==2:
        print >>log, "No input reflection of coordinate files have been given"

    if params.parameters.action=="operator":
      rt_mx = sgtbx.rt_mx(
        params.parameters.apply_operator.user_supplied_operator,t_den=12*8 )
      if params.parameters.inverse:
        rt_mx = rt_mx.inverse()
      print >> log
      print >> log, "Applied operator : ", rt_mx.as_xyz()
      print >> log

      sites = model.sites_frac()
      new_sites = flex.vec3_double()
      for site in sites:
        new_site = rt_mx.r()*matrix.col(site)
        new_site = flex.double(new_site)+flex.double( rt_mx.t().as_double() )
        new_sites.append( tuple(new_site) )
      new_model = model.deep_copy_scatterers()

      new_model.set_sites_frac( new_sites )
      # write the new [pdb file please
      pdb_file = open(params.output.xyzout, 'w')
      print >> log, "Wring pdb file to: %s" % params.output.xyzout
      if params.parameters.apply_operator.concatenate_model:
        write_as_pdb_file(
          input_pdb = pdb_model,
          input_xray_structure = model,
          out = pdb_file,
          chain_id_increment = 0,
          additional_remark = None,
          print_cryst_and_scale=True)

      write_as_pdb_file(
        input_pdb = pdb_model,
        input_xray_structure = new_model,
        out = pdb_file,
        chain_id_increment = params.parameters.chain_id_increment,
        additional_remark = None,
        print_cryst_and_scale=False)

      print >> pdb_file, "END"
      pdb_file.close()

    if params.parameters.action=="manipulate_pdb":
      #rest all the b values
      if params.parameters.manipulate_pdb.set_b:
        b_iso = params.reindex_utils.parameters.manipulate_pdb.b_iso
        new_model = model.set_b_iso( value = b_iso )
        print >> log
        print >> log, "All B-values have been set to %5.3f"%(b_iso)
        print >> log, "Writing PDB file %s"%(params.output.xyzout)
        print >> log

      pdb_file = open(params.output.xyzout, 'w')
      write_as_pdb_file(
        input_pdb = pdb_model,
        input_xray_structure = new_model,
        out = pdb_file,
        chain_id_increment = 0,
        additional_remark = None,
        print_cryst_and_scale=True)
      print >> pdb_file, "END"
      pdb_file.close()

    #write the logfile
    logger = open(params.output.logfile, 'w')
    print >> log, "Writing log file with name %s" % params.output.logfile
    print >> log
    print >> logger, string_buffer.getvalue()
Exemple #47
0
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    print 'Miller set size: %d' % len(miller_set.indices())
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    def nint(a):
        return int(round(a))

    from collections import defaultdict
    i_scale = 0.1
    i_hist = defaultdict(int)

    for j, se in enumerate(zip(starts, ends)):
        s, e = se

        for i in intensi[s:e]:
            i_hist[nint(i_scale * i)] += 1
        
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    fout = open('i_hist.dat', 'w')
    for i in i_hist:
        fout.write('%.2f %d\n' % (i / i_scale, i_hist[i]))
    fout.close()

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    odd = 0
    even = 0

    for s, e in zip(starts, ends):

        for x in range(s, e):
            hkl = lookup[hkl_asu[x]]
            
            if (hkl[0] + hkl[1] + hkl[2]) % 2 == 1:
                odd += 1
            else:
                even += 1
        
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(uc, indices, intensities, sigmas))

    # pre-scale the data - first determine average ln(k), B; then apply

    kbs = [f.kb() for f in frames]

    mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
    mn_B = sum([kb[1] for kb in kbs]) / len(kbs)

    n_lt_500 = 0
    n_gt_500 = 0

    for j, f in enumerate(frames):
        s_i = f.scale_to_kb(mn_k, mn_B)
        fout = open('frame-s-i-%05d.dat' % j, 'w')
        for s, i, si in s_i:
            fout.write('%f %f %f\n' % (s, i, si))
            if i < 500:
                n_lt_500 += 1
            else:
                n_gt_500 += 1
        fout.close()

    from collections import defaultdict

    hist = defaultdict(int)

    fout = open('kb.dat', 'w')

    for j, f in enumerate(frames):
        kb = f.kb()
        fout.write('%4d %6.3f %6.3f\n' % (j, kb[0], kb[1]))
        hist[int(round(kb[1]))] += 1

    fout.close()

    for b in sorted(hist):
        print b, hist[b]


    print odd, even
    print n_lt_500, n_gt_500
    

    return
Exemple #48
0
  def __init__ (self,
                cmd_list,
                nprocs=1,
                out=sys.stdout,
                log=None,
                verbosity=DEFAULT_VERBOSITY,
                output_junit_xml=False) :
    if (log is None) : log = null_out()
    self.out = multi_out()
    self.log = log
    self.out.register("stdout", out)
    self.out.register("log", log)
    self.verbosity = verbosity
    self.quiet = (verbosity == 0)
    self.results = []

    # Filter cmd list for duplicates.
    self.cmd_list = []
    for cmd in cmd_list :
      if (not cmd in self.cmd_list) :
        self.cmd_list.append(cmd)
      else :
        print >> self.out, "Test %s repeated, skipping"%cmd

    # Set number of processors.
    if (nprocs is Auto) :
      nprocs = cpu_count()
    nprocs = min(nprocs, len(self.cmd_list))

    # Starting summary.
    if (self.verbosity > 0) :
      print >> self.out, "Running %d tests on %s processors:"%(len(self.cmd_list), nprocs)
      for cmd in self.cmd_list:
        print >> self.out, "  %s"%cmd
      print >> self.out, ""

    t_start = time.time()
    if nprocs > 1:
      # Run the tests with multiprocessing pool.
      pool = Pool(processes=nprocs)
      for command in self.cmd_list:
        pool.apply_async(
          run_command,
          [command, verbosity, out],
          callback=self.save_result)
      try:
        pool.close()
      except KeyboardInterrupt:
        print >> self.out, "Caught KeyboardInterrupt, terminating"
        pool.terminate()
      finally:
        pool.join()
    else:
      # Run tests serially.
      for command in self.cmd_list:
        rc = run_command(command, verbosity=verbosity, out=out)
        self.save_result(rc)

    # Print ending summary.
    t_end = time.time()
    print >> self.out, "="*80
    print >> self.out, ""
    print >> self.out, "Tests finished. Elapsed time: %.2fs" %(t_end-t_start)
    print >> self.out, ""
    extra_stderr = 0
    test_cases = []
    # Process results for errors and warnings.
    extra_stderr = len([result for result in self.results if result.stderr_lines])
    longjobs = [result for result in self.results if result.wall_time > MAX_TIME]
    warnings = [result for result in self.results if self.check_alert(result) == 1]
    failures = [result for result in self.results if self.check_alert(result) == 2]
    self.finished = len(self.results)
    self.failure = len(failures)
    self.warning = len(warnings)

    # Output JUnit XML
    if output_junit_xml:
      from junit_xml import TestSuite, TestCase
      for result in self.results:
        tc = TestCase(name=result.command,
                      classname=result.command,
                      elapsed_sec=result.wall_time,
                      stdout='\n'.join(result.stdout_lines),
                      stderr='\n'.join(result.stderr_lines))
        if result.return_code != 0:
          tc.add_failure_info(message='exit code %d' %result.return_code)
        #if len(result.stderr_lines):
          #tc.add_error_info(output='\n'.join(result.stderr_lines))
        test_cases.append(tc)
      ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
      with open('output.xml', 'wb') as f:
        print >> f, TestSuite.to_xml_string([ts], prettyprint=True)

    # Run time distribution.
    if (libtbx.env.has_module("scitbx")) :
      from scitbx.array_family import flex
      print >> self.out, "Distribution of test runtimes:"
      hist = flex.histogram(flex.double([result.wall_time for result in self.results]), n_slots=10)
      hist.show(f=self.out, prefix="  ", format_cutoffs="%.1fs")
      print >> self.out, ""

    # Long job warning.
    if longjobs:
      print >> self.out, ""
      print >> self.out, "Warning: the following jobs took at least %d seconds:"%MAX_TIME
      for result in sorted(longjobs, key=lambda result:result.wall_time):
        print >> self.out, "  %s: %.1fs"%(result.command, result.wall_time)
      print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."

    # Failures.
    if failures:
      print >> self.out, ""
      print >> self.out, "Error: the following jobs returned non-zero exit codes or suspicious stderr output:"
      print >> self.out, ""
      for result in warnings:
        self.display_result(result, alert=1, out=self.out, log_return=self.out, log_stderr=self.out)
      for result in failures:
        self.display_result(result, alert=2, out=self.out, log_return=self.out, log_stderr=self.out)
      print >> self.out, ""
      print >> self.out, "Please verify these tests manually."
      print >> self.out, ""

    # Summary
    print >> self.out, "Summary:"
    print >> self.out, "  Tests run                    :",self.finished
    print >> self.out, "  Failures                     :",self.failure
    print >> self.out, "  Warnings (possible failures) :",self.warning
    print >> self.out, "  Stderr output (discouraged)  :",extra_stderr
    if (self.finished != len(self.cmd_list)) :
      print >> self.out, "*" * 80
      print >> self.out, "  WARNING: NOT ALL TESTS FINISHED!"
      print >> self.out, "*" * 80
Exemple #49
0
def sfcalc(args):
    if len(args) == 0:
        print_help()
    elif ("--help" in args):
        print_help()
    elif ("--h" in args):
        print_help()
    elif ("-h" in args):
        print_help()
    else:
        log = multi_out()
        if (not "--quiet" in args):
            log.register(label="stdout", file_object=sys.stdout)
        string_buffer = StringIO()
        string_buffer_plots = StringIO()
        log.register(label="log_buffer", file_object=string_buffer)

        phil_objects = []
        argument_interpreter = master_params.command_line_argument_interpreter(
            home_scope="sfcalc")

        print("#phil __OFF__", file=log)
        print("=================", file=log)
        print("     SFCALC      ", file=log)
        print("=================", file=log)
        print(file=log)

        for arg in args:
            command_line_params = None
            arg_is_processed = False
            # is it a file?
            if (os.path.isfile(arg)):  ## is this a file name?
                # check if it is a phil file
                try:
                    command_line_params = iotbx.phil.parse(file_name=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass
            else:
                try:
                    command_line_params = argument_interpreter.process(arg=arg)
                    if command_line_params is not None:
                        phil_objects.append(command_line_params)
                        arg_is_processed = True
                except KeyboardInterrupt:
                    raise
                except Exception:
                    pass

            if not arg_is_processed:
                print("##----------------------------------------------##",
                      file=log)
                print("## Unknown file or keyword:", arg, file=log)
                print("##----------------------------------------------##",
                      file=log)
                print(file=log)
                raise Sorry("Unknown file or keyword: %s" % arg)

        effective_params = master_params.fetch(sources=phil_objects)
        params = effective_params.extract()

        # now get the unit cell from the files
        hkl_xs = None
        pdb_xs = None

        if params.sfcalc.input.model.file_name is not None:
            pdb_xs = crystal_symmetry_from_any.extract_from(
                file_name=params.sfcalc.input.model.file_name)

        phil_xs = crystal.symmetry(
            unit_cell=params.sfcalc.input.unit_cell,
            space_group_info=params.sfcalc.input.space_group)

        combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs],
                                                      [None])
        combined_xs.show_summary()
        if combined_xs.unit_cell() is None:
            raise Sorry("Unit cell not defined")
        if combined_xs.space_group() is None:
            raise Sorry("Space group not defined")

        # inject the unit cell and symmetry in the phil scope please
        params.sfcalc.input.unit_cell = combined_xs.unit_cell()
        params.sfcalc.input.space_group = \
          sgtbx.space_group_info( group = combined_xs.space_group() )

        print("#phil __ON__", file=log)
        new_params = master_params.format(python_object=params)
        new_params.show(out=log)
        print("#phil __END__", file=log)

        pdb_model = None

        if params.sfcalc.input.model.file_name is not None:
            pdb_model = pdb.input(
                file_name=params.sfcalc.input.model.file_name)
            model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
            print("Atomic model summary", file=log)
            print("====================", file=log)
            model.show_summary()
            print(file=log)

            #make an f_model object please
            b_cart = params.sfcalc.parameters.overall.b_cart
            b_cart = [
                b_cart.b_11, b_cart.b_22, b_cart.b_33, b_cart.b_12,
                b_cart.b_13, b_cart.b_23
            ]
            dummy = abs(
                model.structure_factors(d_min=params.sfcalc.parameters.d_min,
                                        anomalous_flag=False).f_calc())

            flags = dummy.generate_r_free_flags(fraction=0.1,
                                                max_free=99999999)

            fmodel = f_model.manager(
                xray_structure=model,
                r_free_flags=flags,
                target_name="ls_wunit_k1",
                f_obs=dummy,
                b_cart=b_cart,
                k_sol=params.sfcalc.parameters.solvent.k_sol,
                b_sol=params.sfcalc.parameters.solvent.b_sol)

            calc_data_with_solvent_contrib = fmodel.f_model()
            calc_data_with_solvent_contrib = calc_data_with_solvent_contrib.array(
                data=calc_data_with_solvent_contrib.data() *
                params.sfcalc.parameters.overall.k_overall)
            result = None
            label = None
            if params.sfcalc.parameters.output_type == "complex":
                result = calc_data_with_solvent_contrib
                label = "FMODEL"
            if params.sfcalc.parameters.output_type == "amplitudes":
                result = abs(calc_data_with_solvent_contrib)
                label = "FMODEL"
            if params.sfcalc.parameters.output_type == "intensities":
                result = abs(calc_data_with_solvent_contrib)
                result = result.f_as_f_sq()
                label = "IMODEL"

            #write an mtz file with the data
            mtz_dataset = result.as_mtz_dataset(column_root_label=label)

            mtz_dataset.mtz_object().write(
                file_name=params.sfcalc.output.hklout)

        #write the logfile
        logger = open(params.sfcalc.output.logfile, 'w')
        print("writing log file with name %s" % (params.sfcalc.output.logfile),
              file=log)
        print(file=log)
        print(string_buffer.getvalue(), file=logger)
Exemple #50
0
  def __init__ (self,
                cmd_list,
                nprocs=1,
                out=sys.stdout,
                log=None,
                quiet=False,
                output_junit_xml=False) :
    if (log is None) : log = null_out()
    self.out = multi_out()
    self.log = log
    self.out.register("stdout", out)
    self.out.register("log", log)
    self.quiet = quiet
    self.cmd_list = []
    for cmd in cmd_list :
      if (not cmd in self.cmd_list) :
        self.cmd_list.append(cmd)
      else :
        print >> self.out, "  test %s repeated, skipping" % cmd
    nprocs = min(nprocs, len(self.cmd_list))
    print >> self.out, "\n  Starting command list"
    print >> self.out, "    NProcs :",nprocs
    print >> self.out, "    Cmds   :",len(self.cmd_list)
    t_start = time.time()
    if nprocs>1:
      pool = Pool(processes=nprocs)
    self.results = []
    for command in self.cmd_list:
      if nprocs>1:
        pool.apply_async(
          run_command,
          [command, (not quiet), out],
          callback=self.save_result)
      else:
        rc = run_command(command, verbose=(not quiet), out=out)
        self.save_result(rc)
    if nprocs>1:
      try :
        try :
          pool.close()
        except KeyboardInterrupt :
          print >> self.out, "Caught KeyboardInterrupt, terminating"
          pool.terminate()
      finally :
        pool.join()
      print >> self.out, '\nProcesses have joined : %d\n' % len(self.results)
    t_end = time.time()
    print >> self.out, ""
    print >> self.out, "Elapsed time: %.2fs" %(t_end-t_start)
    print >> self.out, ""
    finished = 0
    warning = 0
    extra_stderr = 0
    failure = 0
    failures = []
    long_jobs = []
    long_runtimes = []
    runtimes = []
    if output_junit_xml:
      from junit_xml import TestSuite, TestCase
      test_cases = []
    for result in self.results :
      finished += 1
      runtimes.append(result.wall_time)
      if (result.return_code != 0) :
        failure += 1
        failures.append(result)
      else :
        if (len(result.error_lines) != 0) :
          warning += 1
          failures.append(result)
        if (len(result.stderr_lines) != 0):
          extra_stderr += 1
      if (result.wall_time > max_time) :
        long_jobs.append(result.command)
        long_runtimes.append(result.wall_time)
      if output_junit_xml:
        tc = TestCase(name=result.command,
                      classname=result.command,
                      elapsed_sec=result.wall_time,
                      stdout='\n'.join(result.stdout_lines),
                      stderr='\n'.join(result.stderr_lines))
        if result.return_code != 0:
          tc.add_failure_info(message='exit code %d' %result.return_code)
        #if len(result.stderr_lines):
          #tc.add_error_info(output='\n'.join(result.stderr_lines))
        test_cases.append(tc)

    if output_junit_xml:
      ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
      with open('output.xml', 'wb') as f:
        print >> f, TestSuite.to_xml_string([ts], prettyprint=True)

    if (libtbx.env.has_module("scitbx")) :
      from scitbx.array_family import flex
      print >> self.out, "Distribution of test runtimes:"
      hist = flex.histogram(flex.double(runtimes), n_slots=20)
      hist.show(f=self.out, prefix="  ", format_cutoffs="%.1fs")
      print >> self.out, ""
    if (len(long_jobs) > 0) :
      print >> self.out, ""
      print >> self.out, "WARNING: the following jobs took at least %d seconds:" % \
        max_time
      jobs_and_timings = list(zip(long_jobs, long_runtimes))
      jobs_and_timings.sort(lambda x,y: cmp(x[1], y[1]))
      for cmd, runtime in jobs_and_timings :
        print >> self.out, "  " + cmd + " : %.1fs" % runtime
      print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
    if (len(failures) > 0) :
      print >> self.out, ""
      print >> self.out, "ERROR: the following jobs returned non-zero exit codes or suspicious stderr output:"
      for result in failures :
        print >> self.out, ""
        print >> self.out, result.command + "(exit code %d):" % result.return_code
        for line in result.stderr_lines :
          print >> self.out, "  " + line
        for line in result.error_lines :
          print >> self.out, "  " + line
        print >> self.out, ""
      print >> self.out, "Please verify these tests manually."
      print >> self.out, ""
    print >> self.out, "Summary:"
    print >> self.out, "  Tests run                    :",finished
    print >> self.out, "  Failures                     :",failure
    print >> self.out, "  Warnings (possible failures) :",warning
    print >> self.out, "  Stderr output (discouraged)  :",extra_stderr
    if (finished != len(self.cmd_list)) :
      print >> self.out, "*" * 80
      print >> self.out, "  WARNING: NOT ALL TESTS FINISHED!"
      print >> self.out, "*" * 80
Exemple #51
0
    def __init__(self,
                 cmd_list,
                 nprocs=1,
                 out=sys.stdout,
                 log=None,
                 verbosity=DEFAULT_VERBOSITY,
                 max_time=180):
        if (log is None): log = null_out()
        self.out = multi_out()
        self.log = log
        self.out.register("stdout", out)
        self.out.register("log", log)
        self.verbosity = verbosity
        self.quiet = (verbosity == 0)
        self.results = []

        # Filter cmd list for duplicates.
        self.cmd_list = []
        for cmd in cmd_list:
            if (not cmd in self.cmd_list):
                self.cmd_list.append(cmd)
            else:
                print >> self.out, "Test %s repeated, skipping" % cmd

        # Set number of processors.
        if (nprocs is Auto):
            nprocs = cpu_count()
        nprocs = min(nprocs, len(self.cmd_list))

        # Starting summary.
        if (self.verbosity > 0):
            print >> self.out, "Running %d tests on %s processors:" % (len(
                self.cmd_list), nprocs)
            for cmd in self.cmd_list:
                print >> self.out, "  %s" % cmd
            print >> self.out, ""

        t_start = time.time()
        if nprocs > 1:
            # Run the tests with multiprocessing pool.
            pool = Pool(processes=nprocs)
            for command in self.cmd_list:
                pool.apply_async(run_command, [command, verbosity, out],
                                 callback=self.save_result)
            try:
                pool.close()
            except KeyboardInterrupt:
                print >> self.out, "Caught KeyboardInterrupt, terminating"
                pool.terminate()
            finally:
                pool.join()
        else:
            # Run tests serially.
            for command in self.cmd_list:
                rc = run_command(command, verbosity=verbosity, out=out)
                self.save_result(rc)

        # Print ending summary.
        t_end = time.time()
        print >> self.out, "=" * 80
        print >> self.out, ""
        print >> self.out, "Tests finished. Elapsed time: %.2fs" % (t_end -
                                                                    t_start)
        print >> self.out, ""
        test_cases = []
        # Process results for errors and warnings.
        extra_stderr = len(
            [result for result in self.results if result.stderr_lines])
        longjobs = [
            result for result in self.results if result.wall_time > max_time
        ]
        warnings = [
            result for result in self.results if self.check_alert(result) == 1
        ]
        failures = [
            result for result in self.results if self.check_alert(result) == 2
        ]
        self.finished = len(self.results)
        self.failure = len(failures)
        self.warning = len(warnings)

        # Output JUnit XML if possible
        try:
            from junit_xml import TestSuite, TestCase
            import re

            def decode_string(string):
                try:
                    return string.encode('ascii', 'xmlcharrefreplace')
                except Exception:  # intentional
                    return unicode(string, errors='ignore').encode(
                        'ascii', 'xmlcharrefreplace')

            for result in self.results:
                test_name = reconstruct_test_name(result.command)
                plain_stdout = map(decode_string, result.stdout_lines)
                plain_stderr = map(decode_string, result.stderr_lines)
                output = '\n'.join(plain_stdout + plain_stderr)
                tc = TestCase(classname=test_name[0],
                              name=test_name[1],
                              elapsed_sec=result.wall_time,
                              stdout='\n'.join(plain_stdout),
                              stderr='\n'.join(plain_stderr))
                if result.return_code == 0:
                    # Identify skipped tests
                    if re.search('skip', output, re.IGNORECASE):
                        # find first line including word 'skip' and use it as message
                        skipline = re.search('^((.*)skip(.*))$', output,
                                             re.IGNORECASE
                                             | re.MULTILINE).group(1)
                        tc.add_skipped_info(skipline)
                else:
                    # Test failed. Extract error message and stack trace if possible
                    error_message = 'exit code %d' % result.return_code
                    error_output = '\n'.join(plain_stderr)
                    if plain_stderr:
                        error_message = plain_stderr[-1]
                        if len(plain_stderr) > 20:
                            error_output = '\n'.join(plain_stderr[-20:])
                    tc.add_failure_info(message=error_message,
                                        output=error_output)
                test_cases.append(tc)
            ts = TestSuite("libtbx.run_tests_parallel", test_cases=test_cases)
            with open('output.xml', 'wb') as f:
                print >> f, TestSuite.to_xml_string([ts], prettyprint=True)
        except ImportError, e:
            pass
Exemple #52
0
def run(params):
    if os.path.isdir(params.workdir) and os.listdir(params.workdir):
        print "Directory already exists and not empty:", params.workdir
        return

    if params.reference_file is not None and params.program != "xscale":
        print "WARNING - reference file is not used unless program=xscale."

    if not os.path.isdir(params.workdir):
        os.makedirs(params.workdir)

    if params.batch.engine == "sge":
        batchjobs = batchjob.SGE(pe_name=params.batch.sge_pe_name)
    elif params.batch.engine == "sh":
        batchjobs = batchjob.ExecLocal()
    else:
        raise "Unknown batch engine: %s" % params.batch.engine

    out = multi_out()
    out.register("log", open(os.path.join(params.workdir, "multi_merge.log"), "w"), atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >>out, "Paramters:"
    libtbx.phil.parse(master_params_str).format(params).show(out=out, prefix=" ")
    print >>out, ""

    # XXX Not works when clustering is used..
    html_report = multi_merging.html_report.HtmlReportMulti(os.path.abspath(params.workdir))
    try: html_report.add_params(params, master_params_str)
    except: print >>out, traceback.format_exc()

    xds_ascii_files = map(lambda x: x[:(x.index("#") if "#" in x else None)].strip(), open(params.lstin))
    xds_ascii_files = filter(lambda x: x!="" and os.path.isfile(x), xds_ascii_files)
    xds_ascii_files = map(lambda x: os.path.abspath(x), xds_ascii_files)

    cells = collections.OrderedDict()
    laues = {} # for check
    for xac in xds_ascii_files:
        try:
            symm = XDS_ASCII(xac, read_data=False).symm
        except:
            print >>out, "Error in reading %s" % xac
            print >>out, traceback.format_exc()
            return
        cells[xac] = symm.unit_cell().parameters()
        laue = symm.space_group().build_derived_reflection_intensity_group(False).info()
        laues.setdefault(str(laue),{}).setdefault(symm.space_group_info().type().number(), []).append(xac)

    if len(laues) > 1:
        print >>out, "ERROR! more than one space group included."
        for laue in laues:
            print "Laue symmetry", laue
            for sg in laues[laue]:
                print >>out, " SPACE_GROUP_NUMBER= %d (%d data)" % (sg, len(laues[laue][sg]))
                for f in laues[laue][sg]: print >>out, "  %s" % f
                print >>out, ""
        return
            
    try: html_report.add_cells_and_files(cells, laues.keys()[0])
    except: print >>out, traceback.format_exc()

    data_for_merge = []
    if params.clustering == "blend":
        if params.blend.use_old_result is None:
            blend_wdir = os.path.join(params.workdir, "blend")
            os.mkdir(blend_wdir)
            blend.run_blend0R(blend_wdir, xds_ascii_files)
            print >>out, "\nRunning BLEND with analysis mode"
        else:
            blend_wdir = params.blend.use_old_result
            print >>out, "\nUsing precalculated BLEND result in %s" % params.blend.use_old_result

        blend_clusters = blend.BlendClusters(workdir=blend_wdir, d_min=params.d_min)
        summary_out = os.path.join(blend_wdir, "blend_cluster_summary.dat")
        clusters = blend_clusters.show_cluster_summary(out=open(summary_out, "w"))
        print >>out, "Clusters found by BLEND were summarized in %s" % summary_out

        if params.blend.min_cmpl is not None:
            clusters = filter(lambda x: x[3] >= params.blend.min_cmpl, clusters)
        if params.blend.min_acmpl is not None:
            clusters = filter(lambda x: x[5] >= params.blend.min_acmpl, clusters)            
        if params.blend.min_redun is not None:
            clusters = filter(lambda x: x[4] >= params.blend.min_redun, clusters)
        if params.blend.min_aredun is not None:
            clusters = filter(lambda x: x[6] >= params.blend.min_aredun, clusters)            
        if params.blend.max_LCV is not None:
            clusters = filter(lambda x: x[7] <= params.blend.max_LCV, clusters)
        if params.blend.max_aLCV is not None:
            clusters = filter(lambda x: x[8] <= params.blend.max_aLCV, clusters)

        print >>out, "With specified conditions, following %d clusters will be merged:" % len(clusters)
        for clno, IDs, clh, cmpl, redun, acmpl, aredun, LCV, aLCV in clusters: # process largest first
            print >>out, " Cluster_%.4d NumDS= %4d CLh= %5.1f Cmpl= %6.2f Redun= %4.1f ACmpl=%6.2f ARedun=%4.1f LCV= %5.1f aLCV=%5.1f" % (clno, len(IDs), clh, cmpl, redun, acmpl, aredun, LCV, aLCV)
            data_for_merge.append((os.path.join(params.workdir, "cluster_%.4d"%clno),
                                   map(lambda x: blend_clusters.files[x-1], IDs),
                                   LCV, aLCV,clh))
        print >>out
        try: html_report.add_clutering_result(clusters, "blend")
        except: print >>out, traceback.format_exc()

    elif params.clustering == "cc":
        ccc_wdir = os.path.join(params.workdir, "cc_clustering")
        os.mkdir(ccc_wdir)
        cc_clusters = cc_clustering.CCClustering(ccc_wdir, xds_ascii_files,
                                                 d_min=params.cc_clustering.d_min,
                                                 min_ios=params.cc_clustering.min_ios)
        print >>out, "\nRunning CC-based clustering"

        cc_clusters.do_clustering(nproc=params.cc_clustering.nproc,
                                  b_scale=params.cc_clustering.b_scale,
                                  use_normalized=params.cc_clustering.use_normalized,
                                  html_maker=html_report)
        summary_out = os.path.join(ccc_wdir, "cc_cluster_summary.dat")
        clusters = cc_clusters.show_cluster_summary(d_min=params.d_min, out=open(summary_out, "w"))
        print >>out, "Clusters were summarized in %s" % summary_out

        if params.cc_clustering.min_cmpl is not None:
            clusters = filter(lambda x: x[3] >= params.cc_clustering.min_cmpl, clusters)
        if params.cc_clustering.min_acmpl is not None:
            clusters = filter(lambda x: x[5] >= params.cc_clustering.min_acmpl, clusters)            
        if params.cc_clustering.min_redun is not None:
            clusters = filter(lambda x: x[4] >= params.cc_clustering.min_redun, clusters)
        if params.cc_clustering.min_aredun is not None:
            clusters = filter(lambda x: x[6] >= params.cc_clustering.min_aredun, clusters)            
        if params.cc_clustering.max_clheight is not None:
            clusters = filter(lambda x: x[2] <= params.cc_clustering.max_clheight, clusters)

        print >>out, "With specified conditions, following %d clusters will be merged:" % len(clusters)
        for clno, IDs, clh, cmpl, redun, acmpl, aredun in clusters: # process largest first
            print >>out, " Cluster_%.4d NumDS= %4d CLh= %5.1f Cmpl= %6.2f Redun= %4.1f ACmpl=%6.2f ARedun=%4.1f" % (clno, len(IDs), clh, cmpl, redun, acmpl, aredun)
            data_for_merge.append((os.path.join(params.workdir, "cluster_%.4d"%clno),
                                   map(lambda x: xds_ascii_files[x-1], IDs),
                                   float("nan"),float("nan"),clh))
        print >>out

        try: html_report.add_clutering_result(clusters, "cc_clustering")
        except: print >>out, traceback.format_exc()
        
    else:
        data_for_merge.append((os.path.join(params.workdir, "all_data"),
                               xds_ascii_files, float("nan"), float("nan"), 0))

    ofs_summary = open(os.path.join(params.workdir, "cluster_summary.dat"), "w")
    ofs_summary.write("# d_min= %.3f A\n" % (params.d_min if params.d_min is not None else float("nan")))
    ofs_summary.write("# LCV and aLCV are values of all data\n")
    ofs_summary.write("     cluster  ClH   LCV aLCV run ds.all ds.used  Cmpl Redun I/sigI Rmeas CC1/2 Cmpl.ou Red.ou I/sig.ou Rmeas.ou CC1/2.ou Cmpl.in Red.in I/sig.in Rmeas.in CC1/2.in SigAno.in CCano.in WilsonB Aniso  \n")

    out.flush()

    def write_ofs_summary(workdir, cycle, clh, LCV, aLCV, xds_files, num_files, stats):
        tmps = "%12s %5.2f %4.1f %4.1f %3d %6d %7d %5.1f %5.1f %6.2f %5.1f %5.1f %7.1f %6.1f % 8.2f % 8.1f %8.1f %7.1f %6.1f % 8.2f % 8.1f %8.1f %9.1f %8.1f %7.2f %7.1e\n"
        ofs_summary.write(tmps % (os.path.relpath(workdir, params.workdir), clh, LCV, aLCV, cycle,
                                  len(xds_files), num_files,
                                  stats["cmpl"][0],
                                  stats["redundancy"][0],
                                  stats["i_over_sigma"][0],
                                  stats["r_meas"][0],
                                  stats["cc_half"][0],
                                  stats["cmpl"][2],
                                  stats["redundancy"][2],
                                  stats["i_over_sigma"][2],
                                  stats["r_meas"][2],
                                  stats["cc_half"][2],
                                  stats["cmpl"][1],
                                  stats["redundancy"][1],
                                  stats["i_over_sigma"][1],
                                  stats["r_meas"][1],
                                  stats["cc_half"][1],
                                  stats["sig_ano"][1],
                                  stats["cc_ano"][1],
                                  stats["xtriage_log"].wilson_b,
                                  stats["xtriage_log"].anisotropy,
                                  ))
        ofs_summary.flush()
    # write_ofs_summary()

    if "merging" in params.batch.par_run:
        params.nproc = params.batch.nproc_each
        jobs = []
        for workdir, xds_files, LCV, aLCV, clh in data_for_merge:
            if not os.path.exists(workdir): os.makedirs(workdir)
            shname = "merge_%s.sh" % os.path.relpath(workdir, params.workdir)
            pickle.dump((params, os.path.abspath(workdir), xds_files, cells, batchjobs), open(os.path.join(workdir, "args.pkl"), "w"), -1)
            job = batchjob.Job(workdir, shname, nproc=params.batch.nproc_each)
            job.write_script("""\
"%s" -c '\
import pickle; \
from yamtbx.dataproc.auto.command_line.multi_merge import merge_datasets; \
args = pickle.load(open("args.pkl")); \
ret = merge_datasets(*args); \
pickle.dump(ret, open("result.pkl","w")); \
'
""" % sys.executable)
            batchjobs.submit(job)
            jobs.append(job)

        batchjobs.wait_all(jobs)
        for workdir, xds_files, LCV, aLCV, clh in data_for_merge:
            try:
                results = pickle.load(open(os.path.join(workdir, "result.pkl")))
            except:
                print >>out, "Error in unpickling result in %s" % workdir
                print >>out, traceback.format_exc()
                results = []

            if len(results) == 0:
                ofs_summary.write("#%s failed\n" % os.path.relpath(workdir, params.workdir))
            for cycle, wd, num_files, stats in results:
                write_ofs_summary(workdir, cycle, clh, LCV, aLCV, xds_files, num_files, stats)

            try: html_report.add_merge_result(workdir, clh, LCV, aLCV, xds_files, results[-1][2], results[-1][3])
            except: print >>out, traceback.format_exc()
    else:
        for workdir, xds_files, LCV, aLCV, clh in data_for_merge:
            print >>out, "Merging %s..." % os.path.relpath(workdir, params.workdir)
            out.flush()
            results = merge_datasets(params, workdir, xds_files, cells, batchjobs)
            
            if len(results) == 0:
                ofs_summary.write("#%s failed\n" % os.path.relpath(workdir, params.workdir))

            for cycle, wd, num_files, stats in results:
                write_ofs_summary(workdir, cycle, clh, LCV, aLCV, xds_files, num_files, stats)

            try: html_report.add_merge_result(workdir, clh, LCV, aLCV, xds_files, results[-1][2], results[-1][3])
            except: print >>out, traceback.format_exc()

    try: html_report.write_html()
    except: print >>out, traceback.format_exc()

    print "firefox %s" % os.path.join(html_report.root, "report.html")
    return
Exemple #53
0
def run (args,
    out=sys.stdout,
    program_name="phenix.molprobity",
    ignore_missing_modules=False,
    return_input_objects=False) : # for testing
  rotarama_dir = libtbx.env.find_in_repositories(
    relative_path="chem_data/rotarama_data",
    test=os.path.isdir)
  if (rotarama_dir is None) :
    raise ImportError("Rotamer and Ramachandran distributions not available; "+
      "you will need these to run MolProbity.")
  elif (((not libtbx.env.has_module("reduce")) or
         (not libtbx.env.has_module("probe"))) and
         (not ignore_missing_modules)) :
    raise ImportError("Reduce and/or Probe not configured.")
  import mmtbx.validation.molprobity
  import mmtbx.command_line
  cmdline = mmtbx.command_line.load_model_and_data(
    args=args,
    master_phil=get_master_phil(),
    require_data=False,
    create_fmodel=True,
    process_pdb_file=True,
    usage_string=usage_string,
    prefer_anomalous=True,
    out=out)
  params = cmdline.params
  fmodel = cmdline.fmodel
  if (params.output.maps is Auto) and (fmodel is not None) :
    params.output.maps = True
  elif (params.output.maps == True) and (fmodel is None) :
    raise Sorry("Map output requires experimental data.")
  if (params.molprobity.keep_hydrogens is Auto) :
    params.molprobity.keep_hydrogens = \
      ( (params.input.scattering_table == "neutron") or
        (params.pdb_interpretation.use_neutron_distances) )
  header_info = mmtbx.validation.molprobity.pdb_header_info(
    pdb_file=params.input.pdb.file_name[0],
    pdb_hierarchy=cmdline.pdb_hierarchy)
  pdb_prefix = os.path.splitext(os.path.basename(
    params.input.pdb.file_name[0]))[0]
  if (params.output.prefix is None) :
    params.output.prefix = "molprobity"
  probe_file = None
  if (params.output.probe_dots) or (params.output.kinemage) :
    probe_file = params.output.prefix + "_probe.txt"
  raw_data = cmdline.raw_data

  # check map parameters
  from mmtbx.real_space_correlation import check_map_file
  check_map_file(None, params.input.maps)

  validation = mmtbx.validation.molprobity.molprobity(
    model=cmdline.model,
    fmodel=fmodel,
    flags=params.molprobity.flags,
    sequences=cmdline.sequence,
    raw_data=cmdline.raw_data,
    unmerged_data=cmdline.unmerged_i_obs,
    header_info=header_info,
    keep_hydrogens=params.molprobity.keep_hydrogens,
    nuclear=params.pdb_interpretation.use_neutron_distances,
    save_probe_unformatted_file=probe_file,
    min_cc_two_fofc=params.molprobity.min_cc_two_fofc,
    n_bins_data=params.molprobity.n_bins,
    outliers_only=params.molprobity.outliers_only,
    use_pdb_header_resolution_cutoffs=\
      params.molprobity.use_pdb_header_resolution_cutoffs,
    count_anomalous_pairs_separately=\
      params.molprobity.count_anomalous_pairs_separately,
    use_internal_variance=params.input.unmerged_data.use_internal_variance,
    file_name=params.input.pdb.file_name[0],
    ligand_selection=params.molprobity.ligand_selection,
    rotamer_library=params.molprobity.rotamer_library,
    map_params=params)
  map_file = None

  # model cannot be pickled
  validation.model = None

  # polygon statistics
  validation.polygon_stats = validation.get_polygon_statistics(
    params.polygon.keys_to_show)
  if ('pdb_header_r_work' in params.polygon.keys_to_show):
    validation.polygon_stats['pdb_header_r_work'] = header_info.r_work
  if ('pdb_header_r_free' in params.polygon.keys_to_show):
    validation.polygon_stats['pdb_header_r_free'] = header_info.r_free

  if (not params.output.quiet) :
    out2 = multi_out()
    out2.register("stdout", out)
    f = open(params.output.prefix + ".out", "w")
    out2.register("txt_out", f)
    validation.show(out=out2,
      outliers_only=params.molprobity.outliers_only,
      show_percentiles=params.output.percentiles)
    f.close()
    print >> out, ""
    print >> out, "Results written to %s.out" % params.output.prefix
    if (params.output.kinemage) :
      if (cmdline.pdb_hierarchy.models_size() == 1) :
        assert (probe_file is not None)
        import mmtbx.kinemage.validation
        cmdline.pdb_hierarchy.atoms().reset_i_seq()
        kin_file = "%s.kin" % params.output.prefix
        kin_out = \
          mmtbx.kinemage.validation.export_molprobity_result_as_kinemage(
            result=validation,
            pdb_hierarchy=cmdline.pdb_hierarchy,
            geometry=cmdline.geometry,
            probe_file=probe_file,
            keep_hydrogens=params.molprobity.keep_hydrogens,
            pdbID=pdb_prefix)
        f = open(kin_file, "w")
        f.write(kin_out)
        f.close()
        if (not params.output.quiet) :
          print >> out, "Wrote kinemage to %s" % kin_file
      else :
        print >> out, "Kinemage output not available for multiple MODELs."
    if (params.output.pickle) :
      easy_pickle.dump("%s.pkl" % params.output.prefix, validation)
      if (not params.output.quiet) :
        print >> out, "Saved result to %s.pkl" % params.output.prefix
    if (params.output.coot) :
      coot_file = "%s_coot.py" % params.output.prefix
      validation.write_coot_script(coot_file)
      if (not params.output.quiet) :
        print >> out, "Wrote script for Coot: %s" % coot_file
    if (params.output.maps == True) :
      import mmtbx.maps.utils
      import iotbx.map_tools
      map_file = "%s_maps.mtz" % params.output.prefix
      two_fofc_map, fofc_map = mmtbx.maps.utils.get_maps_from_fmodel(
        fmodel=fmodel,
        fill_missing_f_obs=params.output.map_options.fill_missing_f_obs,
        exclude_free_r_reflections=\
          params.output.map_options.exclude_free_r_reflections)
      anom_map = None
      if (fmodel.f_obs().anomalous_flag()) :
        anom_map = mmtbx.maps.utils.get_anomalous_map(fmodel)
      iotbx.map_tools.write_map_coeffs(
        file_name=map_file,
        fwt_coeffs=two_fofc_map,
        delfwt_coeffs=fofc_map,
        anom_coeffs=anom_map)
      print >> out, "Wrote map coefficients to %s" % map_file
  else :
    print >> out, ""
    validation.show_summary(out=out, show_percentiles=params.output.percentiles)
  if (params.output.wxplots) :
    try :
      import wxtbx.app
    except ImportError, e :
      raise Sorry("wxPython not available.")
    else :
      app = wxtbx.app.CCTBXApp(0)
      validation.display_wx_plots()
      app.MainLoop()
def find_merge_common_images(args):
    phil = iotbx.phil.process_command_line(args = args,
                                           master_string = master_phil).show()
    work_params = phil.work.extract()
    if ("--help" in args) :
        libtbx.phil.parse(master_phil.show())
        return

    if ((work_params.d_min is None) or
        (work_params.data is None) or
        ((work_params.model is None) and
         work_params.scaling.algorithm != "mark1")) :
        raise Usage("cxi.merge "
                    "d_min=4.0 "
                    "data=~/scratch/r0220/006/strong/ "
                    "model=3bz1_3bz2_core.pdb")
    if ((work_params.rescale_with_average_cell) and
        (not work_params.set_average_unit_cell)) :
        raise Usage(
            "If rescale_with_average_cell=True, you must also specify "+
            "set_average_unit_cell=True.")

    # Read Nat's reference model from an MTZ file.  XXX The observation
    # type is given as F, not I--should they be squared?  Check with Nat!
    log = open("%s_%s_scale.log" % (work_params.output.prefix,
                                    work_params.scaling.algorithm), "w")
    out = multi_out()
    out.register("log", log, atexit_send_to=None)
    out.register("stdout", sys.stdout)

    print >> out, "Target unit cell and space group:"
    print >> out, "  ", work_params.target_unit_cell
    print >> out, "  ", work_params.target_space_group

    uc = work_params.target_unit_cell

    miller_set = symmetry(
        unit_cell=work_params.target_unit_cell,
        space_group_info=work_params.target_space_group
        ).build_miller_set(
        anomalous_flag=not work_params.merge_anomalous,
        d_min=work_params.d_min)
    from xfel.cxi.merging.general_fcalc import random_structure
    i_model = random_structure(work_params)

    # ---- Augment this code with any special procedures for x scaling
    scaler = xscaling_manager(
        miller_set=miller_set,
        i_model=i_model,
        params=work_params,
        log=out)
    scaler.read_all()
    print "finished reading"
    sg = miller_set.space_group()
    pg = sg.build_derived_laue_group()
    miller_set.show_summary()

    hkl_asu = scaler.observations["hkl_id"]
    imageno = scaler.observations["frame_id"]
    intensi = scaler.observations["i"]
    sigma_i = scaler.observations["sigi"]
    
    lookup = scaler.millers["merged_asu_hkl"]    

    # construct table of start / end indices for frames: now using Python
    # range indexing

    starts = [0]
    ends = []
    
    for x in xrange(1, len(scaler.observations["hkl_id"])):
        if imageno[x] != imageno[x - 1]:
            ends.append(x)
            starts.append(x)
            
    ends.append(len(scaler.observations["hkl_id"]))

    keep_start = []
    keep_end = []

    for j, se in enumerate(zip(starts, ends)):
        s, e = se
        isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
        dmin = 100.0
        for x in xrange(s, e):
            d = uc.d(lookup[hkl_asu[x]])
            if d < dmin:
                dmin = d
        if isig > 6.0 and dmin < 3.2:
            keep_start.append(s)
            keep_end.append(e)

    starts = keep_start
    ends = keep_end

    print 'Keeping %d frames' % len(starts)

    frames = []

    for s, e in zip(starts, ends):
        indices = [tuple(lookup[hkl_asu[x]]) for x in range(s, e)]
        intensities = intensi[s:e]
        sigmas = sigma_i[s:e]

        frames.append(Frame(uc, indices, intensities, sigmas))

    cycle = 0

    total_nref = sum([len(f.get_indices()) for f in frames])

    # pre-scale the data - first determine average ln(k), B; then apply

    kbs = [f.kb() for f in frames]

    mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
    mn_B = sum([kb[1] for kb in kbs]) / len(kbs)

    for f in frames:
        f.scale_to_kb(mn_k, mn_B)
    
    while True:

        print 'Analysing %d frames' % len(frames)
        print 'Cycle %d' % cycle
        cycle += 1

        print 'Power spectrum'
        fn = frame_numbers(frames)
        for j in sorted(fn):
            print '%4d %4d' % (j, fn[j])
            
        nref_cycle = sum([len(f.get_indices()) for f in frames])
        assert(nref_cycle == total_nref)

        common_reflections = numpy.zeros((len(frames), len(frames)),
                                         dtype = numpy.short)

        obs = { } 

        from cctbx.sgtbx import rt_mx, change_of_basis_op
        oh = change_of_basis_op(rt_mx('h,l,k'))

        for j, f in enumerate(frames):
            indices = set(f.get_indices())
            for i in indices:
                _i = tuple(i)
                if not _i in obs:
                    obs[_i] = []
                obs[_i].append(j)

        # work through unique observations ignoring those which include no
        # hand information
 
        for hkl in obs:
            if hkl == oh.apply(hkl):
                continue
            obs[hkl].sort()
            for j, f1 in enumerate(obs[hkl][:-1]):
                for f2 in obs[hkl][j + 1:]:
                    common_reflections[(f1, f2)] += 1

        cmn_rfl_list = []

        for f1 in range(len(frames)):
            for f2 in range(f1 + 1, len(frames)):
                if common_reflections[(f1, f2)] > 20:
                    cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2))

        cmn_rfl_list.sort()
        cmn_rfl_list.reverse()
    
        joins = []
        used = []
    
        for n, f1, f2 in cmn_rfl_list:

            if f1 in used or f2 in used:
                continue
            
            _cc = frames[f1].cc(frames[f2])

            # really only need to worry about f2 which will get merged...
            # merging multiple files together should be OK provided they are
            # correctly sorted (though the order should not matter anyhow?)
            # anyhow they are sorted anyway... ah as f2 > f1 then just sorting
            # the list by f2 will make sure the data cascase correctly.

            # p-value very small for cc > 0.75 for > 20 observations - necessary
            # as will be correlated due to Wilson curves

            if _cc[0] > 20 and _cc[1] > 0.75:
                print '%4d %.3f' % _cc, f1, f2
                joins.append((f2, f1))
                # used.append(f1)
                used.append(f2)

        if not joins:
            print 'No pairs found'
            break

        joins.sort()
        joins.reverse()
        
        for j2, j1 in joins:
            rmerge = frames[j1].merge(frames[j2])
            if rmerge:
                print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
            else:
                print 'R: %4d %4d ------' % (j1, j2)
                
        continue

    frames.sort()

    print 'Biggest few: #frames; #unique refl'
    j = -1
    while frames[j].get_frames() > 1:
        print frames[j].get_frames(), frames[j].get_unique_indices()
        j -= 1

    return
Exemple #55
0
def set_log(prefix, i_current, i_total):
    log = multi_out()
    fo = open("%s.tls.%s_of_%s" % (prefix, str(i_current), i_total), "w")
    log.register(label="log_buffer", file_object=fo)
    sys.stderr = log
    return log
Exemple #56
0
def run(args):

  if len(args)==0:
    master_params.show(expert_level=0)
  elif ( "--help" in args ):
    print "no help available"
  elif ( "--h" in args ):
    print "no help available"
  elif ( "--show_defaults" in args ):
    master_params.show(expert_level=0)
  elif ( "--show_defaults_all" in args ):
    master_params.show(expert_level=10)



  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    log_plots = StringIO()
    print >> log,"#phil __OFF__"
    print >> log
    print >> log, date_and_time()
    print >> log
    print >> log

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="scaling")

    reflection_file = None

    for arg in args:
      command_line_params = None
      arg_is_processed = False
      if arg == '--quiet':
        arg_is_processed = True
        ## The associated action with this keyword is implemented above
      if (os.path.isfile(arg)): ## is this a file name?
        ## Check if this is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
        except KeyboardInterrupt: raise
        except Exception : pass
        if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        ## Check if this file is a reflection file
        if command_line_params is None:
          reflection_file = reflection_file_reader.any_reflection_file(
            file_name=arg, ensure_read_access=False)
        if (reflection_file is not None):
          reflection_file = arg
          arg_is_processed = True
      ## If it is not a file, it must be a phil command
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown phil-file or phil-command:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file format or phil command: %s" % arg)


    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()

    ## Now please read in the reflections files

    ## get symmetry and cell data first please
    ## By default, the native cell and symmetry are used
    ## as reference
    crystal_symmetry_nat = None
    crystal_symmetry_nat = crystal_symmetry_from_any.extract_from(
      file_name=params.scaling.input.xray_data.after_burn.file_name)

    if params.scaling.input.xray_data.space_group is None:
      params.scaling.input.xray_data.space_group =\
        crystal_symmetry_nat.space_group_info()
      print >> log, "Using symmetry of after_burn data"

    if params.scaling.input.xray_data.unit_cell is None:
      params.scaling.input.xray_data.unit_cell =\
        crystal_symmetry_nat.unit_cell()
      print >> log, "Using cell of after_burn data"

    ## Check if a unit cell is defined
    if params.scaling.input.xray_data.space_group is None:
      raise Sorry("No space group defined")
    if params.scaling.input.xray_data.unit_cell is None:
      raise Sorry("No unit cell defined")


    crystal_symmetry = crystal_symmetry = crystal.symmetry(
      unit_cell =  params.scaling.input.xray_data.unit_cell,
      space_group_symbol = str(
        params.scaling.input.xray_data.space_group) )


    effective_params = master_params.fetch(sources=phil_objects)
    new_params = master_params.format(python_object=params)
    print >> log, "Effective parameters"
    print >> log, "#phil __ON__"
    new_params.show(out=log,
                    expert_level=params.scaling.input.expert_level  )
    print >> log, "#phil __END__"
    print >> log

    ## define a xray data server
    xray_data_server =  reflection_file_utils.reflection_file_server(
      crystal_symmetry = crystal_symmetry,
      force_symmetry = True,
      reflection_files=[])

    ## Read in native data and make appropriatre selections
    miller_array_native = None
    miller_array_native = xray_data_server.get_xray_data(
      file_name = params.scaling.input.xray_data.after_burn.file_name,
      labels = params.scaling.input.xray_data.after_burn.labels,
      ignore_all_zeros = True,
      parameter_scope = 'scaling.input.SIR_scale.xray_data.after_burn'
    )
    info_native = miller_array_native.info()
    miller_array_native=miller_array_native.map_to_asu().select(
      miller_array_native.indices()!=(0,0,0) )
    miller_array_native = miller_array_native.select(
      miller_array_native.data() > 0 )
    ## Convert to amplitudes
    if (miller_array_native.is_xray_intensity_array()):
      miller_array_native = miller_array_native.f_sq_as_f()
    elif (miller_array_native.is_complex_array()):
      miller_array_native = abs(miller_array_native)
    if not miller_array_native.is_real_array():
      raise Sorry("miller_array_native is not a real array")
    miller_array_native.set_info(info = info_native)



    ## Read in derivative data and make appropriate selections
    miller_array_derivative = None
    miller_array_derivative = xray_data_server.get_xray_data(
      file_name = params.scaling.input.xray_data.before_burn.file_name,
      labels = params.scaling.input.xray_data.before_burn.labels,
      ignore_all_zeros = True,
      parameter_scope = 'scaling.input.SIR_scale.xray_data.before_burn'
    )
    info_derivative = miller_array_derivative.info()
    miller_array_derivative=miller_array_derivative.map_to_asu().select(
      miller_array_derivative.indices()!=(0,0,0) )
    miller_array_derivative = miller_array_derivative.select(
      miller_array_derivative.data() > 0 )
    ## Convert to amplitudes
    if (miller_array_derivative.is_xray_intensity_array()):
      miller_array_derivative = miller_array_derivative.f_sq_as_f()
    elif (miller_array_derivative.is_complex_array()):
      miller_array_derivative = abs(miller_array_derivative)
    if not miller_array_derivative.is_real_array():
      raise Sorry("miller_array_derivative is not a real array")
    miller_array_derivative.set_info(info = info_derivative)




    ## As this is a SIR case, we will remove any anomalous pairs
    if miller_array_derivative.anomalous_flag():
      miller_array_derivative = miller_array_derivative.average_bijvoet_mates()\
      .set_observation_type( miller_array_derivative )
    if miller_array_native.anomalous_flag():
      miller_array_native = miller_array_native.average_bijvoet_mates()\
      .set_observation_type( miller_array_native )


    ## Print info
    print >> log
    print >> log, "Native data"
    print >> log, "==========="
    miller_array_native.show_comprehensive_summary(f=log)
    print >> log
    native_pre_scale = pre_scale.pre_scaler(
      miller_array_native,
      params.scaling.input.scaling_strategy.pre_scaler_protocol,
      params.scaling.input.basic)
    miller_array_native =  native_pre_scale.x1.deep_copy()
    del native_pre_scale

    print >> log
    print >> log, "Derivative data"
    print >> log, "==============="
    miller_array_derivative.show_comprehensive_summary(f=log)
    print >> log
    derivative_pre_scale = pre_scale.pre_scaler(
      miller_array_derivative,
      params.scaling.input.scaling_strategy.pre_scaler_protocol,
      params.scaling.input.basic)
    miller_array_derivative =  derivative_pre_scale.x1.deep_copy()
    del derivative_pre_scale

    scaler = fa_estimation.combined_scaling(
      miller_array_native,
      miller_array_derivative,
      params.scaling.input.scaling_strategy.iso_protocol)

    miller_array_native = scaler.x1.deep_copy()
    miller_array_derivative = scaler.x2.deep_copy()
    del scaler

    print >> log
    print >> log, "Making delta f's"
    print >> log, "----------------"
    print >> log

    delta_gen = pair_analyses.delta_generator( miller_array_native,
                                               miller_array_derivative,
                                               params.scaling.input.scaling_strategy.iso_protocol.nsr_bias )
    print >> log
    print >> log, "writing mtz file"
    print >> log, "----------------"
    print >> log

    ## some assertions to make sure nothing went weerd
    assert miller_array_native.observation_type() is not None
    assert miller_array_derivative.observation_type() is not None
    assert delta_gen.abs_delta_f.observation_type() is not None

    ## Please write out the abs_delta_f array

    mtz_dataset = delta_gen.abs_delta_f.as_mtz_dataset(
      column_root_label='F'+params.scaling.input.output.outlabel)
    mtz_dataset.mtz_object().write(
      file_name=params.scaling.input.output.hklout)
Exemple #57
0
    def __init__(self,
                 cmd_list,
                 nprocs=1,
                 out=sys.stdout,
                 log=None,
                 verbosity=DEFAULT_VERBOSITY,
                 max_time=180):
        if (log is None): log = null_out()
        self.out = multi_out()
        self.log = log
        self.out.register("stdout", out)
        self.out.register("log", log)
        self.verbosity = verbosity
        self.quiet = (verbosity == 0)
        self.results = []
        self.pool = None

        # Filter cmd list for duplicates.
        self.cmd_list = []
        for cmd in cmd_list:
            if (not cmd in self.cmd_list):
                self.cmd_list.append(cmd)
            else:
                print >> self.out, "Test %s repeated, skipping" % cmd

        # Set number of processors.
        if (nprocs is Auto):
            nprocs = cpu_count()
        nprocs = min(nprocs, len(self.cmd_list))

        # Starting summary.
        if (self.verbosity > 0):
            print >> self.out, "Running %d tests on %s processors:" % (len(
                self.cmd_list), nprocs)
            for cmd in self.cmd_list:
                print >> self.out, "  %s" % cmd
            print >> self.out, ""

        # Either run tests in parallel or run parallel tests, but
        # can't run parallel tests in parallel (cctbx#95)
        os.environ['OPENBLAS_NUM_THREADS'] = "1"

        t_start = time.time()
        if nprocs > 1:
            # Run the tests with multiprocessing pool.
            self.pool = Pool(processes=nprocs)
            for command in self.cmd_list:
                self.pool.apply_async(run_command, [command, verbosity, out],
                                      callback=self.save_result)
            try:
                self.pool.close()
            except KeyboardInterrupt:
                print >> self.out, "Caught KeyboardInterrupt, terminating"
                self.pool.terminate()
            finally:
                try:
                    self.pool.join()
                except KeyboardInterrupt:
                    pass
        else:
            # Run tests serially.
            for command in self.cmd_list:
                rc = run_command(command, verbosity=verbosity, out=out)
                if self.save_result(rc) == False:
                    break

        # Print ending summary.
        t_end = time.time()
        print >> self.out, "=" * 80
        print >> self.out, ""
        print >> self.out, "Tests finished. Elapsed time: %.2fs" % (t_end -
                                                                    t_start)
        print >> self.out, ""

        # Process results for errors and warnings.
        extra_stderr = len(
            [result for result in self.results if result.stderr_lines])
        longjobs = [
            result for result in self.results if result.wall_time > max_time
        ]
        warnings = [
            result for result in self.results
            if result.alert_status == Status.WARNING
        ]
        failures = [
            result for result in self.results
            if result.alert_status == Status.FAIL
        ]
        self.finished = len(self.results)
        self.failure = len(failures)
        self.warning = len(warnings)

        # Try writing the XML result file
        write_JUnit_XML(self.results, "output.xml")

        # Run time distribution.
        if (libtbx.env.has_module("scitbx")):
            from scitbx.array_family import flex
            print >> self.out, "Distribution of test runtimes:"
            hist = flex.histogram(flex.double(
                [result.wall_time for result in self.results]),
                                  n_slots=10)
            hist.show(f=self.out, prefix="  ", format_cutoffs="%.1fs")
            print >> self.out, ""

        # Long job warning.
        if longjobs:
            print >> self.out, ""
            print >> self.out, "Warning: the following jobs took at least %d seconds:" % max_time
            for result in sorted(longjobs,
                                 key=lambda result: result.wall_time):
                print >> self.out, "  %s: %.1fs" % (result.command,
                                                    result.wall_time)
        else:
            # Just print 5 worst offenders to encourage developers to check them out
            print >> self.out, ""
            print >> self.out, "Warning: the following are 5 longest jobs:"
            for result in sorted(self.results,
                                 key=lambda result: -result.wall_time)[:5]:
                print >> self.out, "  %s: %.1fs" % (result.command,
                                                    result.wall_time)
        print >> self.out, "Please try to reduce overall runtime - consider splitting up these tests."
        print >> self.out, ""

        # Failures.
        if failures:
            print >> self.out, ""
            print >> self.out, "Error: the following jobs returned non-zero exit codes or suspicious stderr output:"
            print >> self.out, ""
            for result in warnings:
                self.display_result(result,
                                    alert=Status.WARNING,
                                    out=self.out,
                                    log_return=self.out,
                                    log_stderr=self.out)
            for result in failures:
                self.display_result(result,
                                    alert=Status.FAIL,
                                    out=self.out,
                                    log_return=self.out,
                                    log_stderr=self.out)
            print >> self.out, ""
            print >> self.out, "Please verify these tests manually."
            print >> self.out, ""

        # Summary
        print >> self.out, "Summary:"
        print >> self.out, "  Tests run                    :", self.finished
        print >> self.out, "  Failures                     :", self.failure
        print >> self.out, "  Warnings (possible failures) :", self.warning
        print >> self.out, "  Stderr output (discouraged)  :", extra_stderr
        if (self.finished != len(self.cmd_list)):
            print >> self.out, "*" * 80
            print >> self.out, "  WARNING: NOT ALL TESTS FINISHED!"
            print >> self.out, "*" * 80
def run(args, command_name="phenix.remove_outliers"):
  if (len(args)==0 or "--help" in args or "--h" in args or "-h" in args):
    print_help(command_name=command_name)
  else:
    log = multi_out()
    plot_out = None
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="outlier_detection")

    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if arg=="--quiet":
        arg_is_processed = True
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()
    if not os.path.exists( params.outlier_utils.input.xray_data.file_name ) :
      raise Sorry("File %s can not be found"%(params.outlier_utils.input.xray_data.file_name) )
    if params.outlier_utils.input.model.file_name is not None:
      if not os.path.exists( params.outlier_utils.input.model.file_name ):
        raise Sorry("File %s can not be found"%(params.outlier_utils.input.model.file_name) )



    # now get the unit cell from the pdb file

    hkl_xs = None
    if params.outlier_utils.input.xray_data.file_name is not None:
      hkl_xs = crystal_symmetry_from_any.extract_from(
        file_name=params.outlier_utils.input.xray_data.file_name)
    pdb_xs = None
    if params.outlier_utils.input.model.file_name is not None:
      pdb_xs = crystal_symmetry_from_any.extract_from(
        file_name=params.outlier_utils.input.model.file_name)

    phil_xs = crystal.symmetry(
      unit_cell=params.outlier_utils.input.unit_cell,
      space_group_info=params.outlier_utils.input.space_group  )

    phil_xs.show_summary()
    hkl_xs.show_summary()


    combined_xs = crystal.select_crystal_symmetry(
      None,phil_xs, [pdb_xs],[hkl_xs])

    # inject the unit cell and symmetry in the phil scope please
    params.outlier_utils.input.unit_cell = combined_xs.unit_cell()
    params.outlier_utils.input.space_group = \
      sgtbx.space_group_info( group = combined_xs.space_group() )

    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)

    if params.outlier_utils.input.unit_cell is None:
      raise Sorry("unit cell not specified")
    if params.outlier_utils.input.space_group is None:
      raise Sorry("space group not specified")
    if params.outlier_utils.input.xray_data.file_name is None:
      raise Sorry("Xray data not specified")
    if params.outlier_utils.input.model.file_name is None:
      print "PDB file not specified. Basic wilson outlier rejections only."



    #-----------------------------------------------------------
    #
    # step 1: read in the reflection file
    #
    phil_xs = crystal.symmetry(
      unit_cell=params.outlier_utils.input.unit_cell,
      space_group_info=params.outlier_utils.input.space_group  )

    xray_data_server =  reflection_file_utils.reflection_file_server(
      crystal_symmetry = phil_xs,
      force_symmetry = True,
      reflection_files=[])

    miller_array = None

    miller_array = xray_data_server.get_xray_data(
      file_name = params.outlier_utils.input.xray_data.file_name,
      labels = params.outlier_utils.input.xray_data.obs_labels,
      ignore_all_zeros = True,
      parameter_scope = 'outlier_utils.input.xray_data',
      parameter_name = 'obs_labels'
      )

    info = miller_array.info()

    miller_array = miller_array.map_to_asu()

    miller_array = miller_array.select(
      miller_array.indices() != (0,0,0))

    #we have to check if the sigma's make any sense at all
    if not miller_array.sigmas_are_sensible():
      miller_array = miller_array.customized_copy(
        data = miller_array.data(),
        sigmas=None).set_observation_type(miller_array)
    miller_array = miller_array.select(
      miller_array.data() > 0 )
    if  miller_array.sigmas() is not None:
      miller_array = miller_array.select(
        miller_array.sigmas() > 0 )

    if (miller_array.is_xray_intensity_array()):
      miller_array = miller_array.f_sq_as_f()
    elif (miller_array.is_complex_array()):
      miller_array = abs(miller_array)

    miller_array.set_info(info)
    merged_anomalous=False
    if miller_array.anomalous_flag():
      miller_array = miller_array.average_bijvoet_mates().set_observation_type(
        miller_array )
      merged_anomalous=True
    miller_array = miller_array.map_to_asu()

    # get the free reflections please
    free_flags = None
    if params.outlier_utils.input.xray_data.free_flags is None:
      free_flags = miller_array.generate_r_free_flags(
         fraction=params.outlier_utils.\
           additional_parameters.free_flag_generation.fraction,
         max_free=params.outlier_utils.\
           additional_parameters.free_flag_generation.max_number,
         lattice_symmetry_max_delta=params.outlier_utils.\
           additional_parameters.free_flag_generation.lattice_symmetry_max_delta,
         use_lattice_symmetry=params.outlier_utils.\
           additional_parameters.free_flag_generation.use_lattice_symmetry
        )
    else:
      free_flags = xray_data_server.get_xray_data(
        file_name = params.outlier_utils.input.xray_data.file_name,
        labels = params.outlier_utils.input.xray_data.free_flags,
        ignore_all_zeros = True,
        parameter_scope = 'outlier_utils.input.xray_data',
        parameter_name = 'free_flags'
        )

      if free_flags.anomalous_flag():
        free_flags = free_flags.average_bijvoet_mates()
        merged_anomalous=True
      free_flags = free_flags.customized_copy(
        data = flex.bool( free_flags.data() == 1 ))
      free_flags = free_flags.map_to_asu()
      free_flags = free_flags.common_set( miller_array )


    print >> log
    print >> log, "Summary info of observed data"
    print >> log, "============================="
    miller_array.show_summary(f=log)
    if merged_anomalous:
      print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged."
    print >> log

    print >> log, "Constructing an outlier manager"
    print >> log, "==============================="
    print >> log
    outlier_manager = outlier_rejection.outlier_manager(
      miller_array,
      free_flags,
      out=log)

    basic_array = None
    extreme_array = None
    model_based_array = None

    basic_array = outlier_manager.basic_wilson_outliers(
      p_basic_wilson = params.outlier_utils.outlier_detection.\
                       parameters.basic_wilson.level,
      return_data = True)

    extreme_array = outlier_manager.extreme_wilson_outliers(
      p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\
                         extreme_wilson.level,
      return_data = True)

    beamstop_array = outlier_manager.beamstop_shadow_outliers(
      level = params.outlier_utils.outlier_detection.parameters.\
               beamstop.level,
      d_min = params.outlier_utils.outlier_detection.parameters.\
               beamstop.d_min,
      return_data=True)



    #----------------------------------------------------------------
    # Step 2: get an xray structure from the PDB file
    #
    if params.outlier_utils.input.model.file_name is not None:
      model = pdb.input(file_name=params.outlier_utils.input.model.file_name).xray_structure_simple(
        crystal_symmetry=phil_xs)
      print >> log, "Atomic model summary"
      print >> log, "===================="
      model.show_summary(f=log)
      print >> log


      # please make an f_model object for bulk solvent scaling etc etc

      f_model_object = f_model.manager(
        f_obs = miller_array,
        r_free_flags = free_flags,
        xray_structure = model )
      print >> log, "Bulk solvent scaling of the data"
      print >> log, "================================"
      print >> log, "Maximum likelihood bulk solvent scaling."
      print >> log
      f_model_object.update_solvent_and_scale(out=log)
      plot_out = StringIO()
      model_based_array = outlier_manager.model_based_outliers(
        f_model_object.f_model(),
        level=params.outlier_utils.outlier_detection.parameters.model_based.level,
        return_data=True,
        plot_out=plot_out)
    #check what needs to be put out please
    if params.outlier_utils.output.hklout is not None:
      if params.outlier_utils.outlier_detection.protocol == "model":
        if params.outlier_utils.input.model.file_name == None:
          print >> log, "Model based rejections requested. No model was supplied."
          print >> log, "Switching to writing out rejections based on extreme value Wilson statistics."
          params.outlier_utils.outlier_detection.protocol="extreme"

      output_array = None
      print >> log
      if params.outlier_utils.outlier_detection.protocol == "basic":
        print >> log, "Non-outliers found by the basic wilson statistics"
        print >> log, "protocol will be written out."
        output_array = basic_array
        new_set_of_free_flags = free_flags.common_set( basic_array )

      if params.outlier_utils.outlier_detection.protocol == "extreme":
        print >> log, "Non-outliers found by the extreme value wilson statistics"
        print >> log, "protocol will be written out."
        output_array = extreme_array
        new_set_of_free_flags = free_flags.common_set( extreme_array )

      if params.outlier_utils.outlier_detection.protocol == "model":
        print >> log, "Non-outliers found by the model based"
        print >> log, "protocol will be written out to the file:"
        print >> log, params.outlier_utils.output.hklout
        print >> log
        output_array = model_based_array
        new_set_of_free_flags = free_flags.common_set( model_based_array )

      if params.outlier_utils.outlier_detection.protocol == "beamstop":
        print >> log, "Outliers found for the beamstop shadow"
        print >> log, "problems detection protocol will be written to the file:"
        print >> log, params.outlier_utils.output.hklout
        print >> log
        output_array = model_based_array
        new_set_of_free_flags = free_flags.common_set( model_based_array )

      mtz_dataset = output_array.as_mtz_dataset(
        column_root_label="FOBS")
      mtz_dataset = mtz_dataset.add_miller_array(
        miller_array = new_set_of_free_flags,
        column_root_label = "Free_R_Flag"
        )
      mtz_dataset.mtz_object().write(
        file_name=params.outlier_utils.output.hklout)

    if (params.outlier_utils.output.logfile is not None):
      final_log = StringIO()
      print >> final_log, string_buffer.getvalue()
      print >> final_log
      if plot_out is not None:
        print >> final_log, plot_out.getvalue()
      outfile = open( params.outlier_utils.output.logfile, 'w' )
      outfile.write( final_log.getvalue() )
      print >> log
      print >> log, "A logfile named %s was created."%(
        params.outlier_utils.output.logfile)
      print >> log, "This logfile contains the screen output and"
      print >> log, "(possibly) some ccp4 style loggraph plots"
Exemple #59
0
    def __init__(self,
                 datatypes=None,
                 phil=None,
                 custom_options=None,
                 logger=None):
        '''
    Base DataManager class
    '''

        self.datatypes = datatypes
        if self.datatypes is None:
            self.datatypes = []
        if hasattr(self, 'datatype') and (self.datatype not in self.datatypes):
            self.datatypes.append(self.datatype)

        # custom data processing options
        self.custom_options = custom_options
        if self.custom_options is not None:
            for option in self.custom_options:
                if option not in data_manager_options:
                    raise Sorry('''\
{option} is not a valid custom option for the DataManager. The available
options are {options}.\
'''.format(option=option, options=', '.join(data_manager_options)))
        else:
            self.custom_options = []

        # functions for custom PHIL
        self.add_custom_phil_str = 'add_%s_phil_str'
        self.export_custom_phil_extract = 'export_%s_phil_extract'
        self.load_custom_phil_extract = 'load_%s_phil_extract'

        # dynamically construct master PHIL string
        self.master_phil_str = 'data_manager {\n'
        for datatype in self.datatypes:

            # check if a datatype has a custom PHIL str
            if hasattr(self, self.add_custom_phil_str % datatype):
                custom_phil_str = getattr(self, self.add_custom_phil_str %
                                          datatype)()
                self.master_phil_str += custom_phil_str

            # default PHIL
            else:
                # sequence_files = None
                #   .type = path
                #   .multiple = True
                self.master_phil_str += '%s_files = None\n' % datatype
                self.master_phil_str += '.type = path\n.multiple=True\n'

                # property for wx GUI (will be removed)
                file_type = any_file_type.get(datatype, None)
                if file_type is not None:
                    self.master_phil_str += '.style = file_type:%s input_file\n' % \
                                                      file_type

            # default_sequence = None
            #   .type = path
            self.master_phil_str += 'default_%s = None\n' % datatype
            self.master_phil_str += '.type = path\n'
        self.master_phil_str += '}'

        self.master_phil = libtbx.phil.parse(self.master_phil_str)

        self._storage = '_%ss'
        self._default = '_default_%s'
        self._current_storage = None
        self._current_default = None

        # generate storage for each data type
        for datatype in self.datatypes:
            # data attributes for each data type
            # e.g self._models, self._default_model
            self._set_datatype(datatype)
            setattr(self, self._current_storage, OrderedDict())
            setattr(self, self._current_default, None)

        # track output files for internal use
        self._output_files = []
        self._output_types = []

        # set defaults
        self._default_output_filename = 'cctbx_program'
        self._overwrite = False
        self._used_output_ext = set()

        # set program
        self._program = None

        # logger (currently used for models)
        self.logger = logger
        if self.logger is None:
            self.logger = multi_out()

        # load information from phil
        if phil is not None:
            self.load_phil_scope(phil)
Exemple #60
0
def sfcalc(args):
  if len(args)==0:
    print_help()
  elif ( "--help" in args ):
    print_help()
  elif ( "--h" in args ):
    print_help()
  elif ("-h" in args ):
    print_help()
  else:
    log = multi_out()
    if (not "--quiet" in args):
      log.register(label="stdout", file_object=sys.stdout)
    string_buffer = StringIO()
    string_buffer_plots = StringIO()
    log.register(label="log_buffer", file_object=string_buffer)

    phil_objects = []
    argument_interpreter = master_params.command_line_argument_interpreter(
      home_scope="sfcalc")

    print >> log, "#phil __OFF__"
    print >> log, "================="
    print >> log, "     SFCALC      "
    print >> log, "================="
    print >> log


    for arg in args:
      command_line_params = None
      arg_is_processed = False
      # is it a file?
      if (os.path.isfile(arg)): ## is this a file name?
        # check if it is a phil file
        try:
          command_line_params = iotbx.phil.parse(file_name=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass
      else:
        try:
          command_line_params = argument_interpreter.process(arg=arg)
          if command_line_params is not None:
            phil_objects.append(command_line_params)
            arg_is_processed = True
        except KeyboardInterrupt: raise
        except Exception : pass

      if not arg_is_processed:
        print >> log, "##----------------------------------------------##"
        print >> log, "## Unknown file or keyword:", arg
        print >> log, "##----------------------------------------------##"
        print >> log
        raise Sorry("Unknown file or keyword: %s" % arg)

    effective_params = master_params.fetch(sources=phil_objects)
    params = effective_params.extract()

    # now get the unit cell from the files
    hkl_xs = None
    pdb_xs = None

    if params.sfcalc.input.model.file_name is not None:
      pdb_xs = crystal_symmetry_from_any.extract_from(
        file_name=params.sfcalc.input.model.file_name)

    phil_xs = crystal.symmetry(
      unit_cell=params.sfcalc.input.unit_cell,
      space_group_info=params.sfcalc.input.space_group  )

    combined_xs = crystal.select_crystal_symmetry(
      None,phil_xs, [pdb_xs],[None])
    combined_xs.show_summary()
    if combined_xs.unit_cell() is None:
      raise Sorry("Unit cell not defined")
    if combined_xs.space_group() is None:
      raise Sorry("Space group not defined")


    # inject the unit cell and symmetry in the phil scope please
    params.sfcalc.input.unit_cell = combined_xs.unit_cell()
    params.sfcalc.input.space_group = \
      sgtbx.space_group_info( group = combined_xs.space_group() )

    print >> log, "#phil __ON__"
    new_params =  master_params.format(python_object=params)
    new_params.show(out=log)
    print >> log, "#phil __END__"

    pdb_model = None

    if params.sfcalc.input.model.file_name is not None:
      pdb_model = pdb.input(file_name=params.sfcalc.input.model.file_name)
      model = pdb_model.xray_structure_simple(crystal_symmetry=phil_xs)
      print >> log, "Atomic model summary"
      print >> log, "===================="
      model.show_summary()
      print >> log

      #make an f_model object please
      b_cart = params.sfcalc.parameters.overall.b_cart
      b_cart = [b_cart.b_11,
                b_cart.b_22,
                b_cart.b_33,
                b_cart.b_12,
                b_cart.b_13,
                b_cart.b_23]
      dummy = abs(model.structure_factors(
        d_min          = params.sfcalc.parameters.d_min,
        anomalous_flag = False).f_calc())

      flags = dummy.generate_r_free_flags(fraction = 0.1,
                                          max_free = 99999999)

      fmodel = f_model.manager( xray_structure   = model,
                                r_free_flags     = flags,
                                target_name      = "ls_wunit_k1",
                                f_obs            = dummy,
                                b_cart           = b_cart,
                                k_sol            = params.sfcalc.parameters.solvent.k_sol,
                                b_sol            = params.sfcalc.parameters.solvent.b_sol )

      calc_data_with_solvent_contrib = fmodel.f_model()
      calc_data_with_solvent_contrib = calc_data_with_solvent_contrib.array(
        data=calc_data_with_solvent_contrib.data()*params.sfcalc.parameters.overall.k_overall)
      result = None
      label = None
      if params.sfcalc.parameters.output_type == "complex":
        result = calc_data_with_solvent_contrib
        label="FMODEL"
      if params.sfcalc.parameters.output_type == "amplitudes":
        result = abs(calc_data_with_solvent_contrib)
        label="FMODEL"
      if params.sfcalc.parameters.output_type == "intensities":
        result = abs(calc_data_with_solvent_contrib)
        result = result.f_as_f_sq()
        label="IMODEL"


      #write an mtz file with the data
      mtz_dataset = result.as_mtz_dataset(
        column_root_label=label)

      mtz_dataset.mtz_object().write(
        file_name=params.sfcalc.output.hklout)


    #write the logfile
    logger = open( params.sfcalc.output.logfile, 'w')
    print >> log, "writing log file with name %s"%(params.sfcalc.output.logfile)
    print >> log
    print >> logger, string_buffer.getvalue()