def run(args): from libtbx.option_parser import option_parser command_line = (option_parser( usage="fable.python %s [options]" % __file__).option( None, "--ifort", action="store_true", default=False).option(None, "--verbose", action="store_true", default=False)).process(args=args) keys = set(command_line.args) exercises = set() for key in globals().keys(): if (key.startswith("exercise_")): exercises.add(key[9:]) assert len(keys) == 0 or keys.issubset(exercises) co = command_line.options from libtbx.utils import show_times_at_exit show_times_at_exit() if (len(keys) == 0 or "open" in keys): exercise_open(opts=co) if (len(keys) == 0 or "mixed_read_write" in keys): exercise_mixed_read_write(opts=co) if (len(keys) == 0 or "read_from_non_existing_file" in keys): exercise_read_from_non_existing_file(opts=co) print("OK")
def run(): opt_parser = (option_parser( usage=""" clean_clutter [-t n | --tabsize=n] file1 file2 ... clean_clutter [-t n | --tabsize=n] [directory] clean_clutter [-t n | --tabsize=n] [--committing|-c]""", description="""The first form cleans the specified files whereas the second form cleans all files in the hierarchy rooted in the given directory or the current directory is none is given. The -c options restricts cleaning to those files which would be committed by running svn commit.""") .option("-t", "--tabsize", action="store", type="int", default=8, help="the number of spaces a tab is to be replaced by", metavar="INT") .option("-c", "--committing", action="store_true", default=False, help="whether to clean the files which are to be committed") ) command_line = opt_parser.process(args=sys.argv[1:]) co = command_line.options files = command_line.args if co.committing and files: opt_parser.show_help() exit(1) if co.committing: try: files = list(subversion.marked_for_commit()) except RuntimeError, err: print err exit(1)
def run(args): command_line = (option_parser() .option("--output_dirname", "-o", type="string", help="Directory for output files.") ).process(args=args) args = command_line.args output_dirname = command_line.options.output_dirname if output_dirname is None: output_dirname = os.path.dirname(args[0]) assert len(args) == 2 xy_pairs = [] for i, filename in enumerate(args): print "Reading data from: %s" %filename f = open(filename, 'rb') x, y = zip(*[line.split() for line in f.readlines() if not line.startswith("#")]) x = flex.double(flex.std_string(x)) y = flex.double(flex.std_string(y)) xy_pairs.append((x,y)) signal = xy_pairs[0] background = xy_pairs[1] signal_x, background_subtracted = subtract_background(signal, background, plot=True) filename = os.path.join(output_dirname, "background_subtracted.txt") f = open(filename, "wb") print >> f, "\n".join(["%i %f" %(x, y) for x, y in zip(signal_x, background_subtracted)]) f.close() print "Background subtracted spectrum written to %s" %filename
def run(args): log = sys.stdout if (len(args) == 0): args = ["--help"] command_line = (option_parser(usage="%s [options] pdb_file" % libtbx.env.dispatcher_name).option( None, "--buffer_layer", action="store", type="float", default=5)).process(args=args, nargs=1) pdb_inp = iotbx.pdb.input(file_name=command_line.args[0]) atoms = pdb_inp.atoms() box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center( sites_cart=atoms.extract_xyz(), buffer_layer=command_line.options.buffer_layer) atoms.set_xyz(new_xyz=box.sites_cart) print >> log, 'REMARK %s --buffer-layer=%.6g %s' % ( libtbx.env.dispatcher_name, command_line.options.buffer_layer, show_string(command_line.args[0])) print >> log, 'REMARK %s' % date_and_time() iotbx.pdb.write_whole_pdb_file( output_file=log, pdb_hierarchy=pdb_inp.construct_hierarchy(), crystal_symmetry=box.crystal_symmetry(), ss_annotation=pdb_inp.extract_secondary_structure(log=null_out()))
def run(args): from libtbx.option_parser import option_parser command_line = (option_parser( usage="fable.python %s [options] regex_pattern ..." % __file__) .enable_multiprocessing() .option(None, "--dry_run", action="store_true", default=False) .option(None, "--valgrind", action="store_true", default=False) .option(None, "--ifort", action="store_true", default=False) .option(None, "--keep_going", action="store_true", default=False) .option(None, "--pch", action="store_true", default=False) .option(None, "--verbose", action="store_true", default=False) ).process(args=args) from libtbx.utils import show_times_at_exit show_times_at_exit() n_failures = exercise_compile_valid( regex_patterns=command_line.args, opts=command_line.options) if (n_failures != 0): print "Done." else: print "OK"
def run(args): if (len(args) == 0): args = ["--help"] from libtbx.option_parser import option_parser import libtbx.load_env command_line = (option_parser( usage="%s [options] pdb_file" % libtbx.env.dispatcher_name) .option(None, "--buffer_layer", action="store", type="float", default=5) ).process(args=args, nargs=1) import iotbx.pdb pdb_inp = iotbx.pdb.input(file_name=command_line.args[0]) atoms = pdb_inp.atoms() from cctbx import uctbx box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center( sites_cart=atoms.extract_xyz(), buffer_layer=command_line.options.buffer_layer) atoms.set_xyz(new_xyz=box.sites_cart) from libtbx.str_utils import show_string print 'REMARK %s --buffer-layer=%.6g %s' % ( libtbx.env.dispatcher_name, command_line.options.buffer_layer, show_string(command_line.args[0])) from libtbx.utils import date_and_time print 'REMARK %s' % date_and_time() print iotbx.pdb.format_cryst1_record(crystal_symmetry=box.crystal_symmetry()) print pdb_inp.construct_hierarchy().as_pdb_string(append_end=True),
def run(args): if (len(args) == 0): args = ["--help"] command_line = (option_parser(usage="%s [OPTIONS] FILE..." % libtbx.env.dispatcher_name).option( None, "--niggli_cell", action="store_true")).process(args=args) if (len(command_line.args) == 0): command_line.parser.show_help() return co = command_line.options for arg in command_line.args: crystal_symmetry = crystal_symmetry_from_any.extract_from(arg) if (crystal_symmetry is None): raise RuntimeError, \ "Unknown file format or unit cell and space group missing from file." if (co.niggli_cell and crystal_symmetry.unit_cell() is not None and crystal_symmetry.space_group_info() is not None): crystal_symmetry = crystal_symmetry.niggli_cell() format.crystal_symmetry(crystal_symmetry) print print "\n".join( crystal_symmetry_as_cns_inp_defines( crystal_symmetry=crystal_symmetry)) print print format_cryst1_and_scale_records( crystal_symmetry=crystal_symmetry, write_scale_records=True) print
def run(args): if (len(args) == 0): args = ["--help"] command_line = (option_parser( usage="%s [OPTIONS] FILE..." % libtbx.env.dispatcher_name) .option(None, "--niggli_cell", action="store_true") ).process(args=args) if (len(command_line.args) == 0): command_line.parser.show_help() return co = command_line.options for arg in command_line.args: crystal_symmetry = crystal_symmetry_from_any.extract_from(arg) if (crystal_symmetry is None): raise RuntimeError, \ "Unknown file format or unit cell and space group missing from file." if (co.niggli_cell and crystal_symmetry.unit_cell() is not None and crystal_symmetry.space_group_info() is not None): crystal_symmetry = crystal_symmetry.niggli_cell() format.crystal_symmetry(crystal_symmetry) print print "\n".join( crystal_symmetry_as_cns_inp_defines(crystal_symmetry=crystal_symmetry)) print print format_cryst1_and_scale_records( crystal_symmetry=crystal_symmetry, write_scale_records=True) print
def run(args): from libtbx.option_parser import option_parser command_line = (option_parser( usage="fable.python %s [options] regex_pattern ..." % __file__).enable_multiprocessing().option( None, "--dry_run", action="store_true", default=False).option( None, "--valgrind", action="store_true", default=False).option( None, "--ifort", action="store_true", default=False).option( None, "--keep_going", action="store_true", default=False).option( None, "--pch", action="store_true", default=False).option( None, "--verbose", action="store_true", default=False)).process(args=args) from libtbx.utils import show_times_at_exit show_times_at_exit() n_failures = exercise_compile_valid(regex_patterns=command_line.args, opts=command_line.options) if (n_failures != 0): print "Done." else: print "OK"
def run(args): command_line = (option_parser().option( "--output_dirname", "-o", type="string", help="Directory for output files.")).process(args=args) args = command_line.args output_dirname = command_line.options.output_dirname if output_dirname is None: output_dirname = os.path.dirname(args[0]) assert len(args) == 2 xy_pairs = [] for i, filename in enumerate(args): print "Reading data from: %s" % filename f = open(filename, 'rb') x, y = zip(*[ line.split() for line in f.readlines() if not line.startswith("#") ]) x = flex.double(flex.std_string(x)) y = flex.double(flex.std_string(y)) xy_pairs.append((x, y)) signal = xy_pairs[0] background = xy_pairs[1] signal_x, background_subtracted = subtract_background(signal, background, plot=True) filename = os.path.join(output_dirname, "background_subtracted.txt") f = open(filename, "wb") print >> f, "\n".join( ["%i %f" % (x, y) for x, y in zip(signal_x, background_subtracted)]) f.close() print "Background subtracted spectrum written to %s" % filename
def run(args): if (len(args) == 0): args = ["--help"] from libtbx.option_parser import option_parser import libtbx.load_env command_line = (option_parser( usage="%s [options] fortran_file ..." % libtbx.env.dispatcher_name) .option(None, "--each", action="store_true", default=False) .option(None, "--report_success", action="store_true", default=False) .option(None, "--warnings", action="store_true", default=False) ).process(args=args) co = command_line.options def sorry_exclusive(opt_name): from libtbx.utils import Sorry raise Sorry( "%s: options are mutually exclusive: --each, --%s" % (libtbx.env.dispatcher_name, opt_name)) if (co.each): if (co.warnings): sorry_exclusive("warnings") from fable.read import process if (co.each): process_each( process=process, file_names=command_line.args, report_success=co.report_success) else: all_fprocs = process(file_names=command_line.args) if (co.warnings): for fproc in all_fprocs.all_in_input_order: report_equivalence_clusters_with_mixed_data_types(fproc=fproc)
def run(args): from libtbx.option_parser import option_parser command_line = (option_parser().option( None, "--comprehensive", action="store", type="string", default='').option( None, "--languages", action="store", type="string", default='fem,fortran').option( None, "--implementations", action="store", type="string", default='dgesvd,dgesdd')).process(args=sys.argv[1:]) exercise() comprehensive = command_line.options.comprehensive use_fortran_flags = [{ 'fem': False, 'fortran': True }[l] for l in command_line.options.languages.split(',')] svd_impl_names = command_line.options.implementations.split(',') for svd_impl_name in svd_impl_names: for use_fortran in use_fortran_flags: compare_times(svd_impl_name=svd_impl_name, use_fortran=use_fortran, comprehensive=comprehensive) print "OK"
def run(): libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--fix_random_seeds", action="store_true", default=False) .option(None, "--runs", type='int', default=1) .option(None, "--verbose", action="store_true", default=False) .option(None, "--skip-twin-test", dest='skip_twin_test', action="store_true", default=False) ).process(args=sys.argv[1:]) if command_line.options.fix_random_seeds: flex.set_random_seed(1) random.seed(2) n_runs = command_line.options.runs if n_runs > 1: refinement_test.ls_cycle_repeats = n_runs exercise_normal_equations() exercise_floating_origin_dynamic_weighting(command_line.options.verbose) special_positions_test(n_runs).run() if not command_line.options.skip_twin_test: twin_test().run()
def run(): libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--fix_random_seeds", action="store_true", default=False) .option(None, "--runs", type='int', default=1) .option(None, "--verbose", action="store_true", default=False) ).process(args=sys.argv[1:]) if command_line.options.fix_random_seeds: flex.set_random_seed(1) random.seed(2) n_runs = command_line.options.runs if n_runs > 1: refinement_test.ls_cycle_repeats = n_runs exercise_normal_equations() exercise_floating_origin_dynamic_weighting(command_line.options.verbose) special_positions_test(n_runs).run() twin_test().run()
def run (args) : if (len(args) == 0) or ("--help" in args) : raise Usage(""" eltbx.show_fp_fdp.py --elements=S,CL --wavelength=1.54 eltbx.show_fp_fdp.py --elements=S,CL --source=CuA1""") parser = option_parser() parser.add_option('--elements') parser.add_option('--xray_source') parser.add_option('--wavelength') options, args = parser.parse_args(args) if (options.xray_source is not None) : print "Source: %s" % options.xray_source elif (options.wavelength is not None) : print "Wavelength: %g Angstrom" % float(options.wavelength) print for element in options.elements.split(','): print "Element: %s" % element fdp = [] for table_name, table in (('Henke et al.', henke.table), ('Sasaki et al.', sasaki.table)): inelastic_scattering = table(element) if (options.xray_source) : fp_fdp = inelastic_scattering.at_angstrom( wavelengths.characteristic(options.xray_source).as_angstrom()) elif (options.wavelength) : fp_fdp = inelastic_scattering.at_angstrom(float(options.wavelength)) else : raise Sorry("Either --xray_source=... or --wavelength=... required") print " %-14s: f'=%-9.6g, f''=%-9.6f" % ( table_name, fp_fdp.fp(), fp_fdp.fdp()) fdp.append(fp_fdp.fdp()) print " diff f''=%.2f %%" % ((fdp[1] - fdp[0])/(fdp[1] + fdp[0]) * 100)
def run(args): log = sys.stdout if (len(args) == 0): args = ["--help"] command_line = (option_parser(usage="%s [options] pdb_file" % libtbx.env.dispatcher_name).option( None, "--buffer_layer", action="store", type="float", default=5)).process(args=args, nargs=1) pdb_inp = iotbx.pdb.input(file_name=command_line.args[0]) model = mmtbx.model.manager(model_input=pdb_inp) box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center( sites_cart=model.get_sites_cart(), buffer_layer=command_line.options.buffer_layer) model.set_sites_cart(box.sites_cart) # Bad hack, never repeat. In fact, all the boxing functionality should # go into mmtbx.model.manager model._crystal_symmetry = box.crystal_symmetry() print('REMARK %s --buffer-layer=%.6g %s' % (libtbx.env.dispatcher_name, command_line.options.buffer_layer, show_string(command_line.args[0])), file=log) print('REMARK %s' % date_and_time(), file=log) print(model.model_as_pdb(), file=log)
def run(args): if (len(args) == 0): args = ["--help"] from libtbx.option_parser import option_parser import libtbx.load_env command_line = (option_parser( usage="%s [options] fortran_file ..." % libtbx.env.dispatcher_name).option( None, "--each", action="store_true", default=False).option(None, "--report_success", action="store_true", default=False).option( None, "--warnings", action="store_true", default=False)).process(args=args) co = command_line.options def sorry_exclusive(opt_name): from libtbx.utils import Sorry raise Sorry("%s: options are mutually exclusive: --each, --%s" % (libtbx.env.dispatcher_name, opt_name)) if (co.each): if (co.warnings): sorry_exclusive("warnings") from fable.read import process if (co.each): process_each(process=process, file_names=command_line.args, report_success=co.report_success) else: all_fprocs = process(file_names=command_line.args) if (co.warnings): for fproc in all_fprocs.all_in_input_order: report_equivalence_clusters_with_mixed_data_types(fproc=fproc)
def run(args): from libtbx.option_parser import option_parser command_line = (option_parser( usage="fable.python %s [options]" % __file__) .option(None, "--ifort", action="store_true", default=False) .option(None, "--verbose", action="store_true", default=False) ).process(args=args) keys = set(command_line.args) exercises = set() for key in globals().keys(): if (key.startswith("exercise_")): exercises.add(key[9:]) assert len(keys) == 0 or keys.issubset(exercises) co = command_line.options from libtbx.utils import show_times_at_exit show_times_at_exit() if (len(keys) == 0 or "open" in keys): exercise_open(opts=co) if (len(keys) == 0 or "mixed_read_write" in keys): exercise_mixed_read_write(opts=co) if (len(keys) == 0 or "read_from_non_existing_file" in keys): exercise_read_from_non_existing_file(opts=co) print "OK"
def run(): libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser().option( None, "--verbose", action="store_true").option( None, "--scatterers", dest='n_scatterers', type="int", default=5).option(None, "--resolution", type="float", default=0.2)).process(args=sys.argv[1:]) exercise_ls_restraints(command_line.options)
def run(args): command_line = (option_parser().option( "-o", "--output_filename", action="store", type="string", help="Filename for the output cbf file", default="gain_map.cbf" ).option( "-m", "--metrology", action="store", type="string", help="CBF or DEF file", default=None ).option( "-d", "--distance", action="store", type="int", help= "Detector distance put into the gain cbf file. Not needed for processing.", default="0" ).option( "-w", "--wavelength", action="store", type="float", help= "Incident beam wavelength put into the gain cbf file. Not needed for processing.", default="0")).process(args=args) output_filename = command_line.options.output_filename metrology = command_line.options.metrology assert metrology is not None and os.path.isfile(metrology) args = command_line.args assert len(args) == 1 if args[0].endswith('.txt') or args[0].endswith('.gain'): raw_data = numpy.loadtxt(args[0]) assert raw_data.shape in [(5920, 388), (11840, 194)] tiles = convert_detector(raw_data) else: raise Usage( "Gain input file should be a text file with extension .txt or .gain" ) metro = cbf_file_to_basis_dict(metrology) write_cspad_cbf(tiles, metro, 'cbf', None, output_filename, command_line.options.wavelength, command_line.options.distance)
def run(): opt_parser = (option_parser( usage=""" clean_clutter [-t n | --tabsize=n] file1 file2 ... clean_clutter [-t n | --tabsize=n] [directory] clean_clutter [-t n | --tabsize=n] [--committing|-c]""", description="""The first form cleans the specified files whereas the second form cleans all files in the hierarchy rooted in the given directory or the current directory is none is given. The -c options restricts cleaning to those files which would be committed by running svn commit.""") .option("-t", "--tabsize", action="store", type="int", default=8, help="the number of spaces a tab is to be replaced by", metavar="INT") .option("-c", "--committing", action="store_true", default=False, help="whether to clean the files which are to be committed") ) command_line = opt_parser.process(args=sys.argv[1:]) co = command_line.options files = command_line.args if co.committing and files: opt_parser.show_help() exit(1) run_isort_in_path = False if co.committing: try: files = list(subversion.marked_for_commit()) except RuntimeError as err: print(err) exit(1) else: if len(files) <= 1: if not files: dir = '.' else: dir = files[0] files = [ c.path for c in libtbx.file_clutter.gather([dir]) if c.is_cluttered(flag_x=False) ] if os.path.exists(os.path.join(dir, '.isort.cfg')): run_isort_in_path = dir clean_clutter_in(files, tabsize=co.tabsize) if run_isort_in_path: try: isort(run_isort_in_path) except Exception as e: print("Did not run isort (%s)" % str(e))
def run(args): command_line = ( option_parser() .option( "-o", "--output_filename", action="store", type="string", help="Filename for the output cbf file", default="gain_map.cbf", ) .option("-m", "--metrology", action="store", type="string", help="CBF or DEF file", default=None) .option( "-d", "--distance", action="store", type="int", help="Detector distance put into the gain cbf file. Not needed for processing.", default="0", ) .option( "-w", "--wavelength", action="store", type="float", help="Incident beam wavelength put into the gain cbf file. Not needed for processing.", default="0", ) ).process(args=args) output_filename = command_line.options.output_filename metrology = command_line.options.metrology assert metrology is not None and os.path.isfile(metrology) args = command_line.args assert len(args) == 1 if args[0].endswith(".txt") or args[0].endswith(".gain"): raw_data = numpy.loadtxt(args[0]) assert raw_data.shape in [(5920, 388), (11840, 194)] tiles = convert_detector(raw_data) else: raise Usage("Gain input file should be a text file with extension .txt or .gain") metro = cbf_file_to_basis_dict(metrology) write_cspad_cbf( tiles, metro, "cbf", None, output_filename, command_line.options.wavelength, command_line.options.distance )
def run(): libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--verbose", action="store_true") .option(None, "--scatterers", dest='n_scatterers', type="int", default=5) .option(None, "--resolution", type="float", default=0.2) ).process(args=sys.argv[1:]) exercise_ls_restraints(command_line.options)
def run(args): from libtbx.option_parser import option_parser command_line = ( option_parser() .option(None, "--comprehensive", action="store", type="string", default="") .option(None, "--languages", action="store", type="string", default="fem,fortran") .option(None, "--implementations", action="store", type="string", default="dgesvd,dgesdd") ).process(args=sys.argv[1:]) exercise() comprehensive = command_line.options.comprehensive use_fortran_flags = [{"fem": False, "fortran": True}[l] for l in command_line.options.languages.split(",")] svd_impl_names = command_line.options.implementations.split(",") for svd_impl_name in svd_impl_names: for use_fortran in use_fortran_flags: compare_times(svd_impl_name=svd_impl_name, use_fortran=use_fortran, comprehensive=comprehensive) print "OK"
def run(args, command_name="scitbx.histogram"): command_line = (option_parser( usage=command_name + " [options] [data_file ...]", description="Example: %s my_data --slots=20" % command_name).option( "-s", "--slots", action="store", type="int", default=10, help="number of histogram slots", metavar="INT").option( None, "--min", action="store", type="float", default=None, help="min data value in histogram", metavar="FLOAT").option( None, "--max", action="store", type="float", default=None, help="max data value in histogram", metavar="FLOAT").option( "-f", "--format_cutoffs", action="store", type="str", default="%.8g", help="format specifier for cutoff values", metavar="STR")).process(args=args) def pro(file_object): co = command_line.options process_file(file_object=file_object, n_slots=co.slots, data_min=co.min, data_max=co.max, format_cutoffs=co.format_cutoffs) if (len(command_line.args) == 0): pro(file_object=sys.stdin) else: for file_name in command_line.args: pro(file_object=open(file_name))
def run(): import sys from libtbx.option_parser import option_parser try: import gltbx.gl except ImportError: print "Skipping gltbx/tst_ellipsoids.py: gltbx.gl module not available." sys.exit(1) exercise_ellipsoid() command_line = (option_parser(usage="", description="").option( None, "--time", action="store_true")).process(args=sys.argv[1:]) if command_line.options.time: if command_line.args: time_ellipsoid(int(command_line.args[0])) else: time_ellipsoid() print "OK"
def run(args, command_name="scitbx.histogram"): command_line = (option_parser( usage=command_name+" [options] [data_file ...]", description="Example: %s my_data --slots=20" % command_name) .option("-s", "--slots", action="store", type="int", default=10, help="number of histogram slots", metavar="INT") .option(None, "--min", action="store", type="float", default=None, help="min data value in histogram", metavar="FLOAT") .option(None, "--max", action="store", type="float", default=None, help="max data value in histogram", metavar="FLOAT") .option("-f", "--format_cutoffs", action="store", type="str", default="%.8g", help="format specifier for cutoff values", metavar="STR") ).process(args=args) def pro(file_object): co = command_line.options process_file( file_object=file_object, n_slots=co.slots, data_min=co.min, data_max=co.max, format_cutoffs=co.format_cutoffs) if (len(command_line.args) == 0): pro(file_object=sys.stdin) else: for file_name in command_line.args: pro(file_object=open(file_name))
def run(args, command_name="libtbx.list_files"): if (len(args) == 0): args = ["."] command_line = (option_parser( usage="%s [options] path ..." % command_name, description="Recursively lists all files," " excluding CVS and .svn directories and .pyc files.").option( "-t", "--text", action="store_true", default=False, help="list text files only").option( "-b", "--binary", action="store_true", default=False, help="list binary files only").option( "-q", "--quote", action="store_true", default=False, help="quote file names")).process(args=args) paths = command_line.args co = command_line.options text = co.text binary = co.binary quote = co.quote if (not (text or binary)): binary = True text = True if (len(paths) == 0): paths = ["."] for path in paths: if (not os.path.exists(path)): print("No such file or directory:", path, file=sys.stderr) elif (os.path.isfile(path)): show_status(path=path, text=text, binary=binary, quote=quote) else: for file_path in walk_source_tree(top=path): show_status(path=file_path, text=text, binary=binary, quote=quote)
def run(): import libtbx.utils libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser().option( None, "--normal_eqns_solving_method", default='naive').option(None, "--fix_random_seeds", action='store_true', default='naive')).process(args=sys.argv[1:]) opts = command_line.options if opts.fix_random_seeds: import random random.seed(1) flex.set_random_seed(1) gradient_threshold = 1e-8 step_threshold = 1e-8 if opts.normal_eqns_solving_method == 'naive': m = lambda eqns: normal_eqns_solving.naive_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=step_threshold) elif opts.normal_eqns_solving_method == 'levenberg-marquardt': m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=gradient_threshold, tau=1e-7) else: raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method) for t in [ saturated_test_case(m), sucrose_test_case(m), symmetry_equivalent_test_case(m), fpfdp_test_case(m), constrained_fpfdp_test_case(m), scalar_scaled_adp_test_case(m), ]: t.run()
def run(): import sys from libtbx.option_parser import option_parser try: import gltbx.gl except ImportError: print "Skipping gltbx/tst_ellipsoids.py: gltbx.gl module not available." sys.exit(1) exercise_ellipsoid() command_line = (option_parser( usage="", description="") .option(None, "--time", action="store_true") ).process(args=sys.argv[1:]) if command_line.options.time: if command_line.args: time_ellipsoid(int(command_line.args[0])) else: time_ellipsoid() print "OK"
def run(args): options = (option_parser() .option(None, '--time_on_file', metavar="PATH", help="time reading and writing." "The file to read shall be a hkl file, i.e " "each line has a format like " "'int int int double double'. " "The end shall be marked by 0 0 0") .option(None, '--buffer_size', type='int', metavar="INT") ).process(args).options for i_trial in range(3): stringio_test_case().run() for i_trial in range(3): cstringio_test_case().run() for i_trial in range(3): mere_file_test_case().run() if options.time_on_file: time_it(options.time_on_file, options.buffer_size) print('OK')
def run(args): options = (option_parser() .option(None, '--time_on_file', metavar="PATH", help="time reading and writing." "The file to read shall be a hkl file, i.e " "each line has a format like " "'int int int double double'. " "The end shall be marked by 0 0 0") .option(None, '--buffer_size', type='int', metavar="INT") ).process(args).options for i_trial in xrange(3): stringio_test_case().run() for i_trial in xrange(3): cstringio_test_case().run() for i_trial in xrange(3): mere_file_test_case().run() if options.time_on_file: time_it(options.time_on_file, options.buffer_size) print 'OK'
def run(): import libtbx.utils libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--normal_eqns_solving_method", default='naive') .option(None, "--fix_random_seeds", action='store_true', default='naive') ).process(args=sys.argv[1:]) opts = command_line.options if opts.fix_random_seeds: import random random.seed(1) flex.set_random_seed(1) gradient_threshold=1e-8 step_threshold=1e-8 if opts.normal_eqns_solving_method == 'naive': m = lambda eqns: normal_eqns_solving.naive_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=step_threshold) elif opts.normal_eqns_solving_method == 'levenberg-marquardt': m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=gradient_threshold, tau=1e-7) else: raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method) for t in [ saturated_test_case(m), sucrose_test_case(m), symmetry_equivalent_test_case(m), ]: t.run()
def run(args): if (len(args) == 0) or ("--help" in args): raise Usage(""" eltbx.show_fp_fdp.py --elements=S,CL --wavelength=1.54 eltbx.show_fp_fdp.py --elements=S,CL --source=CuA1""") parser = option_parser() parser.add_option('--elements') parser.add_option('--xray_source') parser.add_option('--wavelength') options, args = parser.parse_args(args) if (options.xray_source is not None): print("Source: %s" % options.xray_source) elif (options.wavelength is not None): print("Wavelength: %g Angstrom" % float(options.wavelength)) print() for element in options.elements.split(','): print("Element: %s" % element) fdp = [] for table_name, table in (('Henke et al.', henke.table), ('Sasaki et al.', sasaki.table)): inelastic_scattering = table(element) if (options.xray_source): fp_fdp = inelastic_scattering.at_angstrom( wavelengths.characteristic( options.xray_source).as_angstrom()) elif (options.wavelength): fp_fdp = inelastic_scattering.at_angstrom( float(options.wavelength)) else: raise Sorry( "Either --xray_source=... or --wavelength=... required") print(" %-14s: f'=%-9.6g, f''=%-9.6f" % (table_name, fp_fdp.fp(), fp_fdp.fdp())) fdp.append(fp_fdp.fdp()) print(" diff f''=%.2f %%" % ((fdp[1] - fdp[0]) / (fdp[1] + fdp[0]) * 100))
def run(args, command_name="libtbx.list_files"): if (len(args) == 0): args = ["."] command_line = (option_parser( usage="%s [options] path ..." % command_name, description="Recursively lists all files," " excluding CVS and .svn directories and .pyc files.") .option("-t", "--text", action="store_true", default=False, help="list text files only") .option("-b", "--binary", action="store_true", default=False, help="list binary files only") .option("-q", "--quote", action="store_true", default=False, help="quote file names") ).process(args=args) paths = command_line.args co = command_line.options text = co.text binary = co.binary quote = co.quote if (not (text or binary)): binary = True text = True if (len(paths) == 0): paths = ["."] for path in paths: if (not os.path.exists(path)): print >> sys.stderr, "No such file or directory:", path elif (os.path.isfile(path)): show_status(path=path, text=text, binary=binary, quote=quote) else: for file_path in walk_source_tree(top=path): show_status(path=file_path, text=text, binary=binary, quote=quote)
def run(args): if (len(args) == 0): args = ["--help"] from libtbx.option_parser import option_parser import libtbx.load_env command_line = (option_parser(usage="%s [options] fortran_file ..." % libtbx.env.dispatcher_name).option( None, "--top_procedure", action="append", type="str").option( None, "--write_graphviz_dot", action="store", type="str")).process(args=args) co = command_line.options from fable.read import process all_fprocs = process(file_names=command_line.args) topological_fprocs = all_fprocs.build_bottom_up_fproc_list_following_calls( top_procedures=co.top_procedure) dep_cycles = topological_fprocs.dependency_cycles if (len(dep_cycles) != 0): print "Dependency cycles:", len(dep_cycles) for cycle in dep_cycles: print " ", " ".join(cycle) print print "Top-down procedure list:" print digraph_lhs_rhs = [] for fproc in reversed(topological_fprocs.bottom_up_list): if (fproc.name is None): lhs = fproc.fproc_type print lhs else: lhs = fproc.name.value print fproc.fproc_type, fproc.name.value fwds = set( topological_fprocs.forward_uses_by_identifier.get( fproc.name.value, [])) for identifier in sorted(fproc.fdecl_by_identifier.keys()): fdecl = fproc.fdecl_by_identifier[identifier] if (fdecl.is_fproc_name()): continue if (not fdecl.is_user_defined_callable()): continue called_name = fdecl.id_tok.value passed = fproc.externals_passed_by_arg_identifier.get(called_name) if (passed is None): digraph_lhs_rhs.append((lhs, called_name)) else: called_name += "->" + ",".join(sorted(passed)) for indirectly_called_name in passed: digraph_lhs_rhs.append((lhs, indirectly_called_name)) if (fdecl.is_function()): sz = "" if (fdecl.size_tokens is not None): if (len(fdecl.size_tokens) == 1 and fdecl.size_tokens[0].is_integer()): sz = "*%s" % fdecl.size_tokens[0].value else: sz = "*(*)" s = "%s (%s%s)" % (called_name, fdecl.data_type.value, sz) else: s = called_name if (called_name in fwds): s += " (dependency cycle)" print " %s" % s print if (co.write_graphviz_dot is not None): f = open(co.write_graphviz_dot, "w") print >> f, "digraph G {" for lhs_rhs in digraph_lhs_rhs: print >> f, " %s -> %s;" % lhs_rhs print >> f, "}" del f
def run(args, command_name="libtbx.phil", converter_registry=None): if (len(args) == 0): args = ["--help"] command_line = (option_parser( usage="%s [options] parameter_file ..." % command_name).option( None, "--diff", action="store_true", help="Display only differences between the first file (master)" " and the combined definitions from all other files.").option( None, "--show_help", action="store_true", help="Display help for each parameter if available.").option( None, "--show_some_attributes", action="store_true", help="Display non-default attributes for each parameter." ).option( None, "--show_all_attributes", action="store_true", help="Display all attributes for each parameter.").option( None, "--process_includes", action="store_true", help="Inline include files.").option( None, "--print_width", action="store", type="int", help="Width for output", metavar="INT").option( None, "--print_prefix", action="store", type="string", default="", help="Prefix string for output")).process( args=args) co = command_line.options attributes_level = 0 if (co.show_all_attributes): attributes_level = 3 elif (co.show_some_attributes): attributes_level = 2 elif (co.show_help): attributes_level = 1 prefix = co.print_prefix file_names = command_line.args def parse(file_name): return libtbx.phil.parse(file_name=file_name, converter_registry=converter_registry, process_includes=co.process_includes) def show(scope): scope.show(out=sys.stdout, prefix=prefix, attributes_level=attributes_level, print_width=co.print_width) if (not co.diff): for file_name in file_names: print prefix.rstrip() show(scope=parse(file_name=file_name)) print prefix.rstrip() else: if (len(file_names) < 2): raise Sorry("Option --diff requires at least two file names.") master = parse(file_name=file_names[0]) show(scope=master.fetch_diff(sources=[ parse(file_name=file_name) for file_name in file_names[1:] ]))
def run_tests(build_dir, dist_dir, tst_list, display_times=False): if display_times: t0 = time.time() start = time.asctime() libtbx.env.full_testing = True args = [arg.lower() for arg in sys.argv[1:]] command_line = (option_parser( usage="run_tests [-j n]", description="Run several threads in parallel, each picking and then" " running tests one at a time.").option( "-j", "--threads", action="store", type="int", default=1, help="number of threads", metavar="INT").option("-v", "--verbose", action="store_true", default=False).option( "-q", "--quick", action="store_true", default=False).option( "-g", "--valgrind", action="store_true", default=False)).process(args=args, max_nargs=0) co = command_line.options if (threading is None or co.threads == 1): for cmd in iter_tests_cmd(co, build_dir, dist_dir, tst_list): print(cmd) sys.stdout.flush() easy_run.call(command=cmd) print() sys.stderr.flush() sys.stdout.flush() else: cmd_queue = queue.Queue() for cmd in iter_tests_cmd(co, build_dir, dist_dir, tst_list): cmd_queue.put(cmd) threads_pool = [] log_queue = queue.Queue() interrupted = threading.Event() for i in range(co.threads): working_dir = os.tempnam() os.mkdir(working_dir) t = threading.Thread(target=make_pick_and_run_tests( working_dir, interrupted, cmd_queue, log_queue)) threads_pool.append(t) for t in threads_pool: t.start() finished_threads = 0 while (1): try: log = log_queue.get() if isinstance(log, tuple): msg = log[0] print("\n +++ thread %s +++\n" % msg, file=sys.stderr) finished_threads += 1 if finished_threads == co.threads: break else: print(log) except KeyboardInterrupt: print() print("********************************************") print("** Received Keyboard Interrupt **") print("** Waiting for running tests to terminate **") print("********************************************") interrupted.set() break if display_times: print("TIME (%s) (%s) %7.2f %s" % ( start, time.asctime(), time.time() - t0, tst_list, ))
def run(args): if (len(args) == 0): args = ["--help"] from libtbx.option_parser import option_parser import libtbx.load_env command_line = (option_parser( usage="%s [options] fortran_file ..." % libtbx.env.dispatcher_name) .option(None, "--top_procedure", action="append", type="str") .option(None, "--write_graphviz_dot", action="store", type="str") ).process(args=args) co = command_line.options from fable.read import process all_fprocs = process(file_names=command_line.args) topological_fprocs = all_fprocs.build_bottom_up_fproc_list_following_calls( top_procedures=co.top_procedure) dep_cycles = topological_fprocs.dependency_cycles if (len(dep_cycles) != 0): print "Dependency cycles:", len(dep_cycles) for cycle in dep_cycles: print " ", " ".join(cycle) print print "Top-down procedure list:" print digraph_lhs_rhs = [] for fproc in reversed(topological_fprocs.bottom_up_list): if (fproc.name is None): lhs = fproc.fproc_type print lhs else: lhs = fproc.name.value print fproc.fproc_type, fproc.name.value fwds = set( topological_fprocs.forward_uses_by_identifier.get( fproc.name.value, [])) for identifier in sorted(fproc.fdecl_by_identifier.keys()): fdecl = fproc.fdecl_by_identifier[identifier] if (fdecl.is_fproc_name()): continue if (not fdecl.is_user_defined_callable()): continue called_name = fdecl.id_tok.value passed = fproc.externals_passed_by_arg_identifier.get(called_name) if (passed is None): digraph_lhs_rhs.append((lhs, called_name)) else: called_name += "->" + ",".join(sorted(passed)) for indirectly_called_name in passed: digraph_lhs_rhs.append((lhs, indirectly_called_name)) if (fdecl.is_function()): sz = "" if (fdecl.size_tokens is not None): if (len(fdecl.size_tokens) == 1 and fdecl.size_tokens[0].is_integer()): sz = "*%s" % fdecl.size_tokens[0].value else: sz = "*(*)" s = "%s (%s%s)" % (called_name, fdecl.data_type.value, sz) else: s = called_name if (called_name in fwds): s += " (dependency cycle)" print " %s" % s print if (co.write_graphviz_dot is not None): f = open(co.write_graphviz_dot, "w") print >> f, "digraph G {" for lhs_rhs in digraph_lhs_rhs: print >> f, " %s -> %s;" % lhs_rhs print >> f, "}" del f
def run(): command_line = (option_parser( usage="usage: cctbx.euclidean_model_matching [OPTIONS] " "reference_structure.pickle structure.pickle", description="") .option("--tolerance", type="float", default=3) .option("--match_hydrogens", type='bool', default=True) ).process(args=sys.argv[1:]) if len(command_line.args) != 2: command_line.parser.print_help() sys.exit(1) reference_structure = easy_pickle.load(command_line.args[0]) if (type(reference_structure) in (type([]), type(()))): reference_structure = reference_structure[0] structures = easy_pickle.load(command_line.args[1]) if (not type(structures) in (type([]), type(()))): structures = [structures] if not command_line.options.match_hydrogens: reference_structure.select_inplace( ~reference_structure.element_selection('H')) for structure in structures: structure.select_inplace(~structure.element_selection('H')) print "Reference model:" reference_structure.show_summary() print reference_model = reference_structure.as_emma_model() match_list = [] match_histogram = dicts.with_default_value(0) for structure in structures: structure.show_summary() if (hasattr(structure, "info")): print structure.info print sys.stdout.flush() refined_matches = emma.model_matches( reference_model, structure.as_emma_model(), tolerance=command_line.options.tolerance, models_are_diffraction_index_equivalent=False, break_if_match_with_no_singles=True).refined_matches if (len(refined_matches)): refined_matches[0].show() m = len(refined_matches[0].pairs) else: print "No matches" m = 0 match_list.append(match_record(m, structure.scatterers().size())) match_histogram[m] += 1 print sys.stdout.flush() print "match_list:", match_list keys = match_histogram.keys() keys.sort() keys.reverse() print "matches: frequency" sum = 0 for key in keys: v = match_histogram[key] sum += v s = 0 for key in keys: v = match_histogram[key] s += v print " %3d: %3d = %5.1f%%, %5.1f%%" % (key, v, 100.*v/sum, 100.*s/sum) print sys.stdout.flush()
def run(args): import libtbx.load_env if (len(args) == 0): args = ["--help"] elif (args == ["--example"]): args = [ libtbx.env.under_dist(module_name="fable", path="test/valid/sf.f"), "--namespace", "example", "--run"] from libtbx.option_parser import option_parser command_line = (option_parser( usage="%s [options] fortran_file ..." % libtbx.env.dispatcher_name) .option(None, "--compile", action="store_true", default=False) .option(None, "--link", action="store_true", default=False) .option(None, "--run", action="store_true", default=False) .option(None, "--valgrind", action="store_true", default=False) .option(None, "--each", action="store_true", default=False) .option(None, "--top_procedure", action="append", type="str", metavar="IDENTIFIER") .option(None, "--include_guard_suffix", action="store", type="str", metavar="STRING") .option(None, "--dynamic_parameter", action="append", type="str", metavar="STRING", help='example: --dynamic_parameter="int array_size=100"') .option(None, "--fortran_file_comments", action="store_true", default=False) .option(None, "--no_fem_do_safe", action="store_true", default=False) .option(None, "--arr_nd_size_max", action="store", type="int", default=fable.cout.default_arr_nd_size_max, metavar='INTEGER (default: %d)' % fable.cout.default_arr_nd_size_max) .option(None, "--inline_all", action="store_true", default=False) .option(None, "--common_equivalence_simple", action="store", type="str", default="", metavar="STRING", help='comma-separated list of common names') .option(None, "--namespace", action="store", type="str") .option(None, "--separate_cmn_hpp", action="store_true", default=False) .option(None, "--number_of_function_files", action="store", type="int", metavar="INTEGER") .option(None, "--example", action="store_true", default=False) .option(None, "--debug", action="store_true", default=False) ).process(args=args) co = command_line.options if (co.valgrind): co.run = True if (co.run): co.link = True if (co.link): co.compile = True if (not co.each): process(options=co)(file_names=command_line.args) else: from fable.command_line.read import process_each process_each(process=process(options=co), file_names=command_line.args)
from __future__ import division # LIBTBX_SET_DISPATCHER_NAME cxi.hist_finalise # LIBTBX_PRE_DISPATCHER_INCLUDE_SH PHENIX_GUI_ENVIRONMENT=1 # LIBTBX_PRE_DISPATCHER_INCLUDE_SH export PHENIX_GUI_ENVIRONMENT import sys from libtbx.option_parser import option_parser from xfel.cxi.cspad_ana.histogram_finalise import histogram_finalise if __name__ == '__main__': args = sys.argv[1:] assert len(args) > 0 command_line = (option_parser() .option("--output_dirname", "-o", type="string", help="Directory for output files.") .option("--pickle_pattern", type="string", help="regex for matching pickle files.") ).process(args=args) output_dirname = command_line.options.output_dirname pickle_pattern = command_line.options.pickle_pattern runs = command_line.args if output_dirname is None: output_dirname = runs[0] print "Output directory: %s" %output_dirname histogram_finalise(output_dirname, runs, pickle_pattern=pickle_pattern)
def run(args, command_name="libtbx.phil", converter_registry=None): if (len(args) == 0): args = ["--help"] command_line = (option_parser( usage="%s [options] parameter_file ..." % command_name) .option(None, "--diff", action="store_true", help="Display only differences between the first file (master)" " and the combined definitions from all other files.") .option(None, "--show_help", action="store_true", help="Display help for each parameter if available.") .option(None, "--show_some_attributes", action="store_true", help="Display non-default attributes for each parameter.") .option(None, "--show_all_attributes", action="store_true", help="Display all attributes for each parameter.") .option(None, "--process_includes", action="store_true", help="Inline include files.") .option(None, "--print_width", action="store", type="int", help="Width for output", metavar="INT") .option(None, "--print_prefix", action="store", type="string", default="", help="Prefix string for output") ).process(args=args) co = command_line.options attributes_level = 0 if (co.show_all_attributes): attributes_level = 3 elif (co.show_some_attributes): attributes_level = 2 elif (co.show_help): attributes_level = 1 prefix = co.print_prefix file_names = command_line.args def parse(file_name): return libtbx.phil.parse( file_name=file_name, converter_registry=converter_registry, process_includes=co.process_includes) def show(scope): scope.show( out=sys.stdout, prefix=prefix, attributes_level=attributes_level, print_width=co.print_width) if (not co.diff): for file_name in file_names: print prefix.rstrip() show(scope=parse(file_name=file_name)) print prefix.rstrip() else: if (len(file_names) < 2): raise Sorry("Option --diff requires at least two file names.") master = parse(file_name=file_names[0]) show(scope=master.fetch_diff( sources=[parse(file_name=file_name) for file_name in file_names[1:]]))
def run(args): if (len(args) == 0): args = ["--help"] import libtbx.load_env from libtbx.option_parser import option_parser command_line = (option_parser( usage="%s [options] [var=value] [...] [file_listing_commands] [...]" % libtbx.env.dispatcher_name) .option(None, "--dirs", action="store", type="str", help="create a sub-directory for each run.") .option(None, "--force_clean_dirs", action="store_true", help="forces removal of existing directories before creation.") .option(None, "--command", action="append", type="str", help="command to be executed, e.g. 'echo $(MULTI:1-4)'") .option("-j", "--jobs", action="store", type="int", help="maximum number of parallel jobs (default: all CPUs).") ).process(args=args) co = command_line.options # from libtbx import easy_mp import libtbx.introspection import libtbx.utils from libtbx.utils import Sorry from libtbx.str_utils import show_string # files_listing_commands = [] for arg in command_line.args: earg = op.expandvars(arg) flds = earg.split("=", 1) if (len(flds) != 2): files_listing_commands.append(earg) else: k,v = flds os.environ[k] = v # cmd_infos = [] def cmd_infos_append(line): for l in process_dollar_multi(line): index = len(cmd_infos) cmd_infos.append(libtbx.group_args(index=index, cmd=l, log=None)) if (co.command is not None): for line in co.command: cmd_infos_append(line=line) for file_name in files_listing_commands: file_dir = op.dirname(file_name) for line in open(file_name).read().splitlines(): ll = line.lstrip() if (ll.startswith("#")): continue if (ll.startswith("set ")): continue if (ll.startswith("setenv ")): flds = ll.split(None, 2) if (len(flds) == 2): flds.append("") os.environ[flds[1]] = flds[2] continue cmd_infos_append(line=line.replace("$(DIRNAME)", file_dir)) n_proc = min(len(cmd_infos), libtbx.introspection.number_of_processors()) if (co.jobs is not None): n_proc = max(1, min(co.jobs, n_proc)) print "Number of processors:", n_proc print "Number of jobs:", len(cmd_infos) print sys.stdout.flush() show_times = libtbx.utils.show_times(time_start="now") def show_log(cmd_info): need_log = (cmd_info.index != 0 or co.dirs is not None) if (cmd_info.log is None): if (need_log): print "MISSING: output of command with index %03d:" % cmd_info.index print " %s" % cmd_info.cmd elif (not op.isfile(cmd_info.log)): if (need_log): print "MISSING:", cmd_info.log else: lines = open(cmd_info.log).read().splitlines() if (len(lines) > 10): print "@BEGIN" if (len(lines) != 0): print "\n".join(lines) if (len(lines) > 10): print "@END" def show_logs(): for cmd_info in cmd_infos: if (cmd_info.index == 0 and co.dirs is None): continue print "command:", cmd_info.cmd show_log(cmd_info=cmd_info) if (co.dirs is None): for cmd_info in cmd_infos: cmd_info.log = "log%03d" % cmd_info.index libtbx.utils.remove_files(cmd_info.log) if (n_proc < 2): for cmd_info in cmd_infos: print "command:", cmd_info.cmd run_one_cmd(cmd_info=cmd_info) show_log(cmd_info=cmd_info) else: easy_mp.pool_map( processes=n_proc, func=run_one_cmd, args=cmd_infos, chunksize=1) show_logs() else: old_dirs = [] for cmd_info in cmd_infos: d = "%s%03d" % (co.dirs, cmd_info.index) if (op.exists(d)): if (not co.force_clean_dirs): print >> sys.stderr, "exists already: %s" % show_string(d) old_dirs.append(d) cmd_info.log = op.join(d, "log") if (len(old_dirs) != 0): if (not co.force_clean_dirs): raise Sorry( "Please remove the existing directories or files," " or use a different --dirs assignment.") from libtbx.clear_paths \ import remove_or_rename_files_and_directories_if_possible remaining = remove_or_rename_files_and_directories_if_possible( paths=old_dirs) if (len(remaining) != 0): for d in remaining: print >> sys.stderr, \ "unable to remove or rename: %s" % show_string(d) raise Sorry("Failure removing existing directories.") easy_mp.pool_map( processes=n_proc, func=run_in_dir, args=cmd_infos, chunksize=1) show_logs() print show_times() print sys.stdout.flush()
def run_tests(build_dir, dist_dir, tst_list, display_times=False): if display_times: t0=time.time() start = time.asctime() libtbx.env.full_testing = True args = [arg.lower() for arg in sys.argv[1:]] command_line = (option_parser( usage="run_tests [-j n]", description="Run several threads in parallel, each picking and then" " running tests one at a time.") .option("-j", "--threads", action="store", type="int", default=1, help="number of threads", metavar="INT") .option("-v", "--verbose", action="store_true", default=False) .option("-q", "--quick", action="store_true", default=False) .option("-g", "--valgrind", action="store_true", default=False) ).process(args=args, max_nargs=0) co = command_line.options if (threading is None or co.threads == 1): for cmd in iter_tests_cmd(co, build_dir, dist_dir, tst_list): print cmd sys.stdout.flush() easy_run.call(command=cmd) print sys.stderr.flush() sys.stdout.flush() else: cmd_queue = Queue.Queue() for cmd in iter_tests_cmd(co, build_dir, dist_dir, tst_list): cmd_queue.put(cmd) threads_pool = [] log_queue = Queue.Queue() interrupted = threading.Event() for i in xrange(co.threads): working_dir = os.tempnam() os.mkdir(working_dir) t = threading.Thread( target=make_pick_and_run_tests(working_dir, interrupted, cmd_queue, log_queue)) threads_pool.append(t) for t in threads_pool: t.start() finished_threads = 0 while(1): try: log = log_queue.get() if isinstance(log, tuple): msg = log[0] print >> sys.stderr, "\n +++ thread %s +++\n" % msg finished_threads += 1 if finished_threads == co.threads: break else: print log except KeyboardInterrupt: print print "********************************************" print "** Received Keyboard Interrupt **" print "** Waiting for running tests to terminate **" print "********************************************" interrupted.set() break if display_times: print "TIME (%s) (%s) %7.2f %s" % (start, time.asctime(), time.time()-t0, tst_list, )
def run(args): import libtbx.load_env if (len(args) == 0): args = ["--help"] elif (args == ["--example"]): args = [ libtbx.env.under_dist(module_name="fable", path="test/valid/sf.f"), "--namespace", "example", "--run" ] from libtbx.option_parser import option_parser command_line = ( option_parser(usage="%s [options] fortran_file ..." % libtbx.env.dispatcher_name).option( None, "--compile", action="store_true", default=False).option( None, "--link", action="store_true", default=False).option( None, "--run", action="store_true", default=False).option( None, "--valgrind", action="store_true", default=False).option( None, "--each", action="store_true", default=False).option( None, "--top_procedure", action="append", type="str", metavar="IDENTIFIER").option( None, "--include_guard_suffix", action="store", type="str", metavar="STRING"). option( None, "--dynamic_parameter", action="append", type="str", metavar="STRING", help='example: --dynamic_parameter="int array_size=100"').option( None, "--fortran_file_comments", action="store_true", default=False).option( None, "--no_fem_do_safe", action="store_true", default=False).option( None, "--arr_nd_size_max", action="store", type="int", default=fable.cout.default_arr_nd_size_max, metavar='INTEGER (default: %d)' % fable.cout.default_arr_nd_size_max).option( None, "--inline_all", action="store_true", default=False).option( None, "--common_equivalence_simple", action="store", type="str", default="", metavar="STRING", help='comma-separated list of common names'). option(None, "--namespace", action="store", type="str").option( None, "--separate_cmn_hpp", action="store_true", default=False).option(None, "--number_of_function_files", action="store", type="int", metavar="INTEGER").option( None, "--example", action="store_true", default=False).option( None, "--debug", action="store_true", default=False)).process(args=args) co = command_line.options if (co.valgrind): co.run = True if (co.run): co.link = True if (co.link): co.compile = True if (not co.each): process(options=co)(file_names=command_line.args) else: from fable.command_line.read import process_each process_each(process=process(options=co), file_names=command_line.args)
def run(args): assert len(args) > 0 command_line = (option_parser() .option("--roi", type="string", help="Region of interest for summing up histograms" "from neighbouring pixels.") .option("--log_scale", action="store_true", default=False, help="Draw y-axis on a log scale.") .option("--normalise", action="store_true", default=False, help="Normalise by number of member images.") .option("--save", action="store_true", default=False, help="Save each plot as a png.") .option("--start", type="string", help="Starting pixel coordinates") .option("--fit_gaussians", action="store_true", default=False, help="Fit gaussians to the peaks.") .option("--n_gaussians", type="int", default=2, help="Number of gaussians to fit.") .option("--estimated_gain", type="float", default=30, help="The approximate position of the one photon peak.") ).process(args=args) log_scale = command_line.options.log_scale fit_gaussians = command_line.options.fit_gaussians roi = cspad_tbx.getOptROI(command_line.options.roi) normalise = command_line.options.normalise save_image = command_line.options.save starting_pixel = command_line.options.start n_gaussians = command_line.options.n_gaussians estimated_gain = command_line.options.estimated_gain if starting_pixel is not None: starting_pixel = eval(starting_pixel) assert isinstance(starting_pixel, tuple) args = command_line.args path = args[0] window_title = path d = easy_pickle.load(path) args = args[1:] pixels = None if len(args) > 0: pixels = [eval(arg) for arg in args] for pixel in pixels: assert isinstance(pixel, tuple) assert len(pixel) == 2 if roi is not None: summed_hist = {} for i in range(roi[2], roi[3]): for j in range(roi[0], roi[1]): if (i,j) not in summed_hist: summed_hist[(0,0)] = d[(i,j)] else: summed_hist[(0,0)].update(d[(i,j)]) d = summed_hist #if roi is not None: #summed_hist = None #for i in range(roi[2], roi[3]): #for j in range(roi[0], roi[1]): #if summed_hist is None: #summed_hist = d[(i,j)] #else: #summed_hist.update(d[(i,j)]) #title = str(roi) #plot(hist, window_title=window_title, title=title,log_scale=log_scale, #normalise=normalise, save_image=save_image, fit_gaussians=fit_gaussians) #return histograms = pixel_histograms(d, estimated_gain=estimated_gain) histograms.plot( pixels=pixels, starting_pixel=starting_pixel, fit_gaussians=fit_gaussians, n_gaussians=n_gaussians, window_title=window_title, log_scale=log_scale, save_image=save_image)
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from xfel.cxi.cspad_ana import cspad_tbx if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--nproc", "-n", type="int", default=1, dest="nproc", help="Number of processors") .option(None, "--num-images-max", "-N", type="int", default=None, dest="num_images_max", help="Maximum number of frames to average") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 if len(paths) == 1: # test if the iamge is a multi-image from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([paths[0]]) assert len(datablocks) == 1 datablock = datablocks[0] imagesets = datablock.extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] if not imageset.reader().is_single_file_reader(): from libtbx.utils import Usage raise Usage("Supply more than one image") worker = multi_image_worker(command_line, paths[0], imageset) if command_line.options.num_images_max is not None and command_line.options.num_images_max < len( imageset): iterable = range(command_line.options.num_images_max) else: iterable = range(len(imageset)) else: # Multiple images provided worker = single_image_worker(command_line) if command_line.options.num_images_max is not None and command_line.options.num_images_max < len( paths): iterable = paths[:command_line.options.num_images_max] else: iterable = paths if command_line.options.nproc > 1: iterable = splitit(iterable, command_line.options.nproc) from libtbx import easy_mp if command_line.options.nproc == 1: results = [worker(iterable)] else: results = easy_mp.parallel_map(func=worker, iterable=iterable, processes=command_line.options.nproc) nfail = 0 nmemb = 0 for i, (r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img, r_sum_wavelength, size, active_areas, detector_address, beam_center, pixel_size, saturated_value) in enumerate(results): nfail += r_nfail nmemb += r_nmemb if i == 0: max_img = r_max_img sum_distance = r_sum_distance sum_img = r_sum_img ssq_img = r_ssq_img sum_wavelength = r_sum_wavelength else: sel = (r_max_img > max_img).as_1d() max_img.set_selected(sel, r_max_img.select(sel)) sum_distance += r_sum_distance sum_img += r_sum_img ssq_img += r_ssq_img sum_wavelength += r_sum_wavelength # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
from __future__ import division from libtbx import code_analysis from libtbx.option_parser import option_parser def run(args, debug=False): comments = code_analysis.comments(args, debug=debug) print comments.commented_lines, comments.lines,\ round(comments.commented_lines/comments.lines * 100, 1) if __name__ == '__main__': import sys command_line = (option_parser(usage="", description="").option( None, "--debug", dest='debug', action="store_true", default=False)).process(args=sys.argv[1:]) run(command_line.args, **command_line.options.__dict__)
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from scitbx.array_family import flex from xfel.cxi.cspad_ana import cspad_tbx from iotbx.detectors.cspad_detector_formats import reverse_timestamp if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 # Loop over all images and accumulate statistics. nfail = 0 nmemb = 0 for path in paths: if command_line.options.verbose: sys.stdout.write("Processing %s...\n" % path) try: # Promote the image to double-precision floating point type. # All real-valued flex arrays have the as_double() function. d = easy_pickle.load(path) distance = d['DISTANCE'] img = d['DATA'].as_1d().as_double() wavelength = d['WAVELENGTH'] time_tuple = reverse_timestamp(d['TIMESTAMP']) # Warn if the header items across the set of images do not match # up. Note that discrepancies regarding the image size are # fatal. if 'active_areas' in locals(): if (active_areas != d['ACTIVE_AREAS']).count(True) != 0: sys.stderr.write("Active areas do not match\n") else: active_areas = d['ACTIVE_AREAS'] if 'beam_center' in locals(): if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']): sys.stderr.write("Beam centers do not match\n") else: beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']) if 'detector_address' in locals(): if detector_address != d['DETECTOR_ADDRESS']: sys.stderr.write("Detector addresses do not match\n") else: detector_address = d['DETECTOR_ADDRESS'] if 'saturated_value' in locals(): if saturated_value != d['SATURATED_VALUE']: sys.stderr.write("Saturated values do not match\n") else: saturated_value = d['SATURATED_VALUE'] if 'size' in locals(): if size != (d['SIZE1'], d['SIZE2']): sys.stderr.write("Image sizes do not match\n") return 1 else: size = (d['SIZE1'], d['SIZE2']) if size != d['DATA'].focus(): sys.stderr.write("Image size does not match pixel array\n") return 1 if 'pixel_size' in locals(): if pixel_size != d['PIXEL_SIZE']: sys.stderr.write("Pixel sizes do not match\n") return 1 else: if 'PIXEL_SIZE' in d: pixel_size = d['PIXEL_SIZE'] else: pixel_size = None except Exception: try: # Fall back on reading the image with dxtbx, and shoehorn the # extracted information into what would have been found in a # pickle file. XXX This code assumes a monolithic detector! from dxtbx.format.Registry import Registry format_class = Registry.find(path) i = format_class(path) beam = i.get_beam() assert len(i.get_detector()) == 1 detector = i.get_detector()[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data().as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() time_tuple = (i.get_scan().get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) except Exception: nfail += 1 continue # See also event() in xfel.cxi.cspad_ana.average_tbx. Record the # base time as the timestamp of the first image. # # The sum-of-squares image is accumulated using long integers, as # this delays the point where overflow occurs. But really, this # is just a band-aid... if nmemb == 0: max_img = img.deep_copy() sum_distance = distance sum_img = img.deep_copy() ssq_img = flex.pow2(img) sum_wavelength = wavelength sum_time = (0, 0) time_base = time_tuple else: sel = (img > max_img).as_1d() max_img.set_selected(sel, img.select(sel)) sum_distance += distance sum_img += img ssq_img += flex.pow2(img) sum_wavelength += wavelength sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]), sum_time[1] + (time_tuple[1] - time_base[1])) nmemb += 1 # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_timestamp = cspad_tbx.evt_timestamp( (time_base[0] + int(round(sum_time[0] / nmemb)), time_base[1] + int(round(sum_time[1] / nmemb)))) avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of CSPAD cbf images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import option_parser from scitbx.array_family import flex from dxtbx.format.Registry import Registry from xfel.cftbx.detector.cspad_cbf_tbx import cbf_file_to_basis_dict, write_cspad_cbf # from xfel.cxi.cspad_ana import cspad_tbx # from iotbx.detectors.cspad_detector_formats import reverse_timestamp if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 # Loop over all images and accumulate statistics. nfail = 0 nmemb = 0 for path in paths: if command_line.options.verbose: sys.stdout.write("Processing %s...\n" % path) try: # Promote the image to double-precision floating point type. # All real-valued flex arrays have the as_double() function. # Warn if the header items across the set of images do not match # up. Note that discrepancies regarding the image size are # fatal. if not 'reader' in locals(): reader = Registry.find(path) img = reader(path) if 'detector' in locals(): test_detector = img.get_detector() if len(test_detector) != len(detector): sys.stderr.write("Detectors do not have the same number of panels\n") return 1 for t, d in zip(test_detector, detector): if t.get_image_size() != d.get_image_size(): sys.stderr.write("Panel sizes do not match\n") return 1 if t.get_pixel_size() != d.get_pixel_size(): sys.stderr.write("Pixel sizes do not match\n") return 1 if t.get_d_matrix() != d.get_d_matrix(): sys.stderr.write("Detector panels are not all in the same location. The average will use the positions of the first image.\n") detector = test_detector else: detector = img.get_detector() data = [img.get_raw_data()[i].as_1d().as_double() for i in xrange(len(detector))] wavelength = img.get_beam().get_wavelength() distance = flex.mean(flex.double([d.get_directed_distance() for d in detector])) except Exception: nfail += 1 continue # The sum-of-squares image is accumulated using long integers, as # this delays the point where overflow occurs. But really, this # is just a band-aid... if nmemb == 0: max_img = copy.deepcopy(data) sum_distance = distance sum_img = copy.deepcopy(data) ssq_img = [flex.pow2(d) for d in data] sum_wavelength = wavelength metro = cbf_file_to_basis_dict(path) else: sel = [(d > max_d).as_1d() for d, max_d in zip(data, max_img)] for d, max_d, s in zip(data, max_img, sel): max_d.set_selected(s, d.select(s)) sum_distance += distance for d, sum_d in zip(data, sum_img): sum_d += d for d, ssq_d in zip(data, ssq_img): ssq_d += flex.pow2(d) sum_wavelength += wavelength nmemb += 1 # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = [sum_d.as_double() / nmemb for sum_d in sum_img] avg_distance = sum_distance / nmemb avg_wavelength = sum_wavelength / nmemb def make_tiles(data, detector): """ Assemble a tiles dictionary as required by write_cspad_cbf, consisting of 4 arrays of shape 8x185x388. Assumes the order in the data array matches the order of the enumerated detector panels. """ assert len(data) == 64 tiles = {} s, f = 185, 194 for q_id in xrange(4): tiles[0,q_id] = flex.double((flex.grid(s*8, f*2))) for s_id in xrange(8): for a_id in xrange(2): asic_idx = (q_id*16) + (s_id*2) + a_id asic = data[asic_idx] asic.reshape(flex.grid((s, f))) tiles[0, q_id].matrix_paste_block_in_place(asic, s_id*s, a_id*f) tiles[0, q_id].reshape(flex.grid((8, s, f*2))) return tiles # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: tiles = make_tiles(avg_img, detector) write_cspad_cbf(tiles, metro, 'cbf', None, command_line.options.avg_path, avg_wavelength, avg_distance) if command_line.options.max_path is not None: tiles = make_tiles(max_img, detector) write_cspad_cbf(tiles, metro, 'cbf', None, command_line.options.max_path, avg_wavelength, avg_distance) if command_line.options.stddev_path is not None: stddev_img = [ssq_d.as_double() - sum_d.as_double() * avg_d for ssq_d, sum_d, avg_d in zip(ssq_img, sum_img, avg_img)] # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. for stddev_d in stddev_img: stddev_d.set_selected(stddev_d < 0, 0) if nmemb == 1: stddev_img = [flex.sqrt(stddev_d) for stddev_d in stddev_img] else: stddev_img = [flex.sqrt(stddev_d / (nmemb - 1)) for stddev_d in stddev_img] tiles = make_tiles(stddev_img, detector) write_cspad_cbf(tiles, metro, 'cbf', None, command_line.options.stddev_path, avg_wavelength, avg_distance) return 0
def run(args): command_line = (option_parser() .option("-o", "--output_filename", action="store", type="string", help="Filename for the output pickle file", default="gain_map.pickle") .option("-f", "--detector_format_version", action="store", type="string", help="Detector format version to use for generating active areas and laying out tiles", default=None) .option("-m", "--optical_metrology_path", action="store", type="string", help="Path to slac optical metrology file. If not set, use Run 4 metrology", default=None) .option("-d", "--distance", action="store", type="int", help="Detector distance put into the gain pickle file. Not needed for processing.", default="0") .option("-w", "--wavelength", action="store", type="float", help="Incident beam wavelength put into the gain pickle file. Not needed for processing.", default="0") ).process(args=args) output_filename = command_line.options.output_filename detector_format_version = command_line.options.detector_format_version if detector_format_version is None or 'XPP' not in detector_format_version: beam_center_x = None beam_center_y = None else: beam_center_x = 1765 // 2 * 0.11 beam_center_y = 1765 // 2 * 0.11 address, timestamp = address_and_timestamp_from_detector_format_version(detector_format_version) # if no detector format version is provided, make sure to write no address to the image pickle # but CsPadDetector (called later), needs an address, so give it a fake one save_address = address is not None if not save_address: address = "CxiDs1-0|Cspad-0" # time stamp will still be None timestamp = evt_timestamp((timestamp,0)) args = command_line.args assert len(args) == 1 if args[0].endswith('.npy'): data = numpy.load(args[0]) det, active_areas = convert_2x2(data, detector_format_version, address) elif args[0].endswith('.txt') or args[0].endswith('.gain'): raw_data = numpy.loadtxt(args[0]) assert raw_data.shape in [(5920, 388), (11840, 194)] det, active_areas = convert_detector(raw_data, detector_format_version, address, command_line.options.optical_metrology_path) img_diff = det img_sel = (img_diff > 0).as_1d() gain_map = flex.double(img_diff.accessor(), 0) gain_map.as_1d().set_selected(img_sel.iselection(), 1/img_diff.as_1d().select(img_sel)) gain_map /= flex.mean(gain_map.as_1d().select(img_sel)) if not save_address: address = None d = cspad_tbx.dpack(data=gain_map, address=address, active_areas=active_areas, timestamp=timestamp, distance=command_line.options.distance,wavelength=command_line.options.wavelength, beam_center_x = beam_center_x, beam_center_y = beam_center_y) easy_pickle.dump(output_filename, d)
from __future__ import division from libtbx import code_analysis from libtbx.option_parser import option_parser def run(args, debug=False): comments = code_analysis.comments(args, debug=debug) print comments.commented_lines, comments.lines,\ round(comments.commented_lines/comments.lines * 100, 1) if __name__ == '__main__': import sys command_line = (option_parser( usage="", description="") .option(None, "--debug", dest='debug', action="store_true", default=False) ).process(args=sys.argv[1:]) run(command_line.args, **command_line.options.__dict__)
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ if argv is None: argv = sys.argv dxtbx.util.encode_output_as_utf8() progname = os.getenv("LIBTBX_DISPATCHER_NAME") if not progname or progname.endswith(".python"): progname = "%prog" command_line = (option_parser.option_parser( usage= f"{progname} [-v] [-a PATH] [-m PATH] [-s PATH] image1 image2 [image3 ...]" ).option( None, "--average-path", "-a", type="string", default="avg.cbf", dest="avg_path", metavar="PATH", help="Write average image to PATH", ).option( None, "--maximum-path", "-m", type="string", default="max.cbf", dest="max_path", metavar="PATH", help="Write maximum projection image to PATH", ).option( None, "--stddev-path", "-s", type="string", default="stddev.cbf", dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH", ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress", ).option( None, "--nproc", "-n", type="int", default=1, dest="nproc", help="Number of processors", ).option( None, "--num-images-max", "-N", type="int", default=None, dest="num_images_max", help="Maximum number of frames to average", ).option( None, "--skip-images", "-S", type="int", default=None, dest="skip_images", help="Number of images to skip at the start of the dataset", ).option( None, "--mpi", None, type=bool, default=False, dest="mpi", help="Set to enable MPI processing", )).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 experiments = ExperimentListFactory.from_filenames([paths[0]], load_models=False) if len(paths) == 1: worker = multi_image_worker(command_line, paths[0], experiments) iterable = list(range(len(experiments))) else: # Multiple images provided worker = single_image_worker(command_line) iterable = paths if command_line.options.skip_images is not None: if command_line.options.skip_images >= len(iterable): raise Usage("Skipping all the images") iterable = iterable[command_line.options.skip_images:] if (command_line.options.num_images_max is not None and command_line.options.num_images_max < len(iterable)): iterable = iterable[:command_line.options.num_images_max] assert len(iterable) >= 2, "Need more than one image to average" if command_line.options.mpi: try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes iterable = [ i for n, i in enumerate(iterable) if (n + rank) % size == 0 ] ( r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img, r_sum_wavelength, ) = worker(iterable) nfail = np.array([0]) nmemb = np.array([0]) sum_distance = np.array([0.0]) sum_wavelength = np.array([0.0]) comm.Reduce(np.array([r_nfail]), nfail) comm.Reduce(np.array([r_nmemb]), nmemb) comm.Reduce(np.array([r_sum_distance]), sum_distance) comm.Reduce(np.array([r_sum_wavelength]), sum_wavelength) nfail = int(nfail[0]) nmemb = int(nmemb) sum_distance = float(sum_distance[0]) sum_wavelength = float(sum_wavelength[0]) def reduce_image(data, op=MPI.SUM): result = [] for panel_data in data: panel_data = panel_data.as_numpy_array() reduced_data = np.zeros(panel_data.shape).astype( panel_data.dtype) comm.Reduce(panel_data, reduced_data, op=op) result.append(flex.double(reduced_data)) return result max_img = reduce_image(r_max_img, MPI.MAX) sum_img = reduce_image(r_sum_img) ssq_img = reduce_image(r_ssq_img) if rank != 0: return avg_img = tuple(s / nmemb for s in sum_img) else: if command_line.options.nproc == 1: results = [worker(iterable)] else: iterable = splitit(iterable, command_line.options.nproc) results = easy_mp.parallel_map( func=worker, iterable=iterable, processes=command_line.options.nproc) nfail = 0 nmemb = 0 for ( i, ( r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img, r_sum_wavelength, ), ) in enumerate(results): nfail += r_nfail nmemb += r_nmemb if i == 0: max_img = r_max_img sum_distance = r_sum_distance sum_img = r_sum_img ssq_img = r_ssq_img sum_wavelength = r_sum_wavelength else: for p in range(len(sum_img)): sel = (r_max_img[p] > max_img[p]).as_1d() max_img[p].set_selected(sel, r_max_img[p].select(sel)) sum_img[p] += r_sum_img[p] ssq_img[p] += r_ssq_img[p] sum_distance += r_sum_distance sum_wavelength += r_sum_wavelength # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stdout.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = tuple(s.as_double() / nmemb for s in sum_img) avg_distance = sum_distance / nmemb avg_wavelength = sum_wavelength / nmemb expt = experiments[0] expt.load_models() detector = expt.detector h = detector.hierarchy() origin = h.get_local_origin() h.set_local_frame( h.get_local_fast_axis(), h.get_local_slow_axis(), (origin[0], origin[1], -avg_distance), ) expt.beam.set_wavelength(avg_wavelength) assert expt.beam.get_wavelength() == expt.imageset.get_beam( 0).get_wavelength() # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: for n, d in enumerate(detector): fast, slow = d.get_image_size() avg_img[n].resize(flex.grid(slow, fast)) writer = FullCBFWriter(imageset=expt.imageset) cbf = writer.get_cbf_handle(header_only=True) writer.add_data_to_cbf(cbf, data=avg_img) writer.write_cbf(command_line.options.avg_path, cbf=cbf) if command_line.options.max_path is not None: for n, d in enumerate(detector): fast, slow = d.get_image_size() max_img[n].resize(flex.grid(slow, fast)) max_img = tuple(max_img) writer = FullCBFWriter(imageset=expt.imageset) cbf = writer.get_cbf_handle(header_only=True) writer.add_data_to_cbf(cbf, data=max_img) writer.write_cbf(command_line.options.max_path, cbf=cbf) if command_line.options.stddev_path is not None: stddev_img = [] for n, d in enumerate(detector): stddev_img.append(ssq_img[n].as_double() - sum_img[n].as_double() * avg_img[n]) # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img[n].set_selected(stddev_img[n] < 0, 0) if nmemb == 1: stddev_img[n] = flex.sqrt(stddev_img[n]) else: stddev_img[n] = flex.sqrt(stddev_img[n] / (nmemb - 1)) fast, slow = d.get_image_size() stddev_img[n].resize(flex.grid(slow, fast)) stddev_img = tuple(stddev_img) writer = FullCBFWriter(imageset=expt.imageset) cbf = writer.get_cbf_handle(header_only=True) writer.add_data_to_cbf(cbf, data=stddev_img) writer.write_cbf(command_line.options.stddev_path, cbf=cbf) return 0
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from scitbx.array_family import flex from xfel.cxi.cspad_ana import cspad_tbx if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 # Loop over all images and accumulate statistics. nfail = 0 nmemb = 0 if len(paths) == 1: # test if the iamge is a multi-image from dxtbx.format.Registry import Registry from dxtbx.format.FormatMultiImage import FormatMultiImage format_class = Registry.find(paths[0]) if not issubclass(format_class, FormatMultiImage): from libtbx.utils import Usage raise Usage("Supply more than one image") print "Loading image..." i = format_class(paths[0]) print "Loaded" def read_single_image(n): if command_line.options.verbose: sys.stdout.write("Processing %s: %d...\n" % (paths[0], n)) beam = i.get_beam(n) assert len(i.get_detector(n)) == 1 detector = i.get_detector(n)[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data(n).as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() scan = i.get_scan(n) if scan is None: time_tuple = (0, 0) else: time_tuple = (scan.get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas iterable = xrange(i.get_num_images()) else: def read_single_image(path): if command_line.options.verbose: sys.stdout.write("Processing %s...\n" % path) from dxtbx.format.Registry import Registry format_class = Registry.find(path) i = format_class(path) beam = i.get_beam() assert len(i.get_detector()) == 1 detector = i.get_detector()[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data().as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() scan = i.get_scan() if scan is None: time_tuple = (0, 0) else: time_tuple = (scan.get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas iterable = paths for item in iterable: try: #XXX This code assumes a monolithic detector! beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas = \ read_single_image(item) except Exception: nfail += 1 continue # See also event() in xfel.cxi.cspad_ana.average_tbx. Record the # base time as the timestamp of the first image. # # The sum-of-squares image is accumulated using long integers, as # this delays the point where overflow occurs. But really, this # is just a band-aid... if nmemb == 0: max_img = img.deep_copy() sum_distance = distance sum_img = img.deep_copy() ssq_img = flex.pow2(img) sum_wavelength = wavelength sum_time = (0, 0) time_base = time_tuple else: sel = (img > max_img).as_1d() max_img.set_selected(sel, img.select(sel)) sum_distance += distance sum_img += img ssq_img += flex.pow2(img) sum_wavelength += wavelength sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]), sum_time[1] + (time_tuple[1] - time_base[1])) nmemb += 1 # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_timestamp = cspad_tbx.evt_timestamp( (time_base[0] + int(round(sum_time[0] / nmemb)), time_base[1] + int(round(sum_time[1] / nmemb)))) avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import option_parser if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name).option( None, "--average-path", "-a", type="string", default="avg.cbf", dest="avg_path", metavar="PATH", help="Write average image to PATH", ).option( None, "--maximum-path", "-m", type="string", default="max.cbf", dest="max_path", metavar="PATH", help="Write maximum projection image to PATH", ).option( None, "--stddev-path", "-s", type="string", default="stddev.cbf", dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH", ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress", ).option( None, "--nproc", "-n", type="int", default=1, dest="nproc", help="Number of processors", ).option( None, "--num-images-max", "-N", type="int", default=None, dest="num_images_max", help="Maximum number of frames to average", ).option( None, "--skip-images", "-S", type="int", default=None, dest="skip_images", help="Number of images to skip at the start of the dataset", ).option( None, "--mpi", None, type=bool, default=False, dest="mpi", help="Set to enable MPI processing", )).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 if len(paths) == 1: # test if the iamge is a multi-image from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([paths[0]]) assert len(datablocks) == 1 datablock = datablocks[0] imagesets = datablock.extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] if not imageset.reader().is_single_file_reader(): from libtbx.utils import Usage raise Usage("Supply more than one image") worker = multi_image_worker(command_line, paths[0], imageset) iterable = range(len(imageset)) else: # Multiple images provided worker = single_image_worker(command_line) iterable = paths if command_line.options.skip_images is not None: if command_line.options.skip_images >= len(iterable): from libtbx.utils import Usage raise Usage("Skipping all the images") iterable = iterable[command_line.options.skip_images:] if (command_line.options.num_images_max is not None and command_line.options.num_images_max < iterable): iterable = iterable[:command_line.options.num_images_max] assert len(iterable) >= 2, "Need more than one image to average" if len(paths) > 1: from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([iterable[0]]) assert len(datablocks) == 1 datablock = datablocks[0] imagesets = datablock.extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] from libtbx import easy_mp if command_line.options.mpi: try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes iterable = [ iterable[i] for i in xrange(len(iterable)) if (i + rank) % size == 0 ] results = [worker(iterable)] results = comm.gather(results, root=0) if rank != 0: return results_set = [] for r in results: results_set.extend(r) results = results_set else: if command_line.options.nproc == 1: results = [worker(iterable)] else: iterable = splitit(iterable, command_line.options.nproc) results = easy_mp.parallel_map( func=worker, iterable=iterable, processes=command_line.options.nproc) nfail = 0 nmemb = 0 for ( i, ( r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img, r_sum_wavelength, ), ) in enumerate(results): nfail += r_nfail nmemb += r_nmemb if i == 0: max_img = r_max_img sum_distance = r_sum_distance sum_img = r_sum_img ssq_img = r_ssq_img sum_wavelength = r_sum_wavelength else: for p in xrange(len(sum_img)): sel = (r_max_img[p] > max_img[p]).as_1d() max_img[p].set_selected(sel, r_max_img[p].select(sel)) sum_img[p] += r_sum_img[p] ssq_img[p] += r_ssq_img[p] sum_distance += r_sum_distance sum_wavelength += r_sum_wavelength # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = tuple( [sum_img[p].as_double() / nmemb for p in xrange(len(sum_img))]) avg_distance = sum_distance / nmemb avg_wavelength = sum_wavelength / nmemb detector = imageset.get_detector() h = detector.hierarchy() origin = h.get_local_origin() h.set_local_frame( h.get_local_fast_axis(), h.get_local_slow_axis(), (origin[0], origin[1], -avg_distance), ) imageset.get_beam().set_wavelength(avg_wavelength) # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: for p in xrange(len(detector)): fast, slow = detector[p].get_image_size() avg_img[p].resize(flex.grid(slow, fast)) writer = FullCBFWriter(imageset=imageset) cbf = writer.get_cbf_handle(header_only=True) writer.add_data_to_cbf(cbf, data=avg_img) writer.write_cbf(command_line.options.avg_path, cbf=cbf) if command_line.options.max_path is not None: for p in xrange(len(detector)): fast, slow = detector[p].get_image_size() max_img[p].resize(flex.grid(slow, fast)) max_img = tuple(max_img) writer = FullCBFWriter(imageset=imageset) cbf = writer.get_cbf_handle(header_only=True) writer.add_data_to_cbf(cbf, data=max_img) writer.write_cbf(command_line.options.max_path, cbf=cbf) if command_line.options.stddev_path is not None: stddev_img = [] for p in xrange(len(detector)): stddev_img.append(ssq_img[p].as_double() - sum_img[p].as_double() * avg_img[p]) # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img[p].set_selected(stddev_img[p] < 0, 0) if nmemb == 1: stddev_img[p] = flex.sqrt(stddev_img[p]) else: stddev_img[p] = flex.sqrt(stddev_img[p] / (nmemb - 1)) fast, slow = detector[p].get_image_size() stddev_img[p].resize(flex.grid(slow, fast)) stddev_img = tuple(stddev_img) writer = FullCBFWriter(imageset=imageset) cbf = writer.get_cbf_handle(header_only=True) writer.add_data_to_cbf(cbf, data=stddev_img) writer.write_cbf(command_line.options.stddev_path, cbf=cbf) return 0
def run(): command_line = (option_parser( usage="usage: cctbx.euclidean_model_matching [OPTIONS] " "reference_structure.pickle structure.pickle", description="").option("--tolerance", type="float", default=3).option( "--match_hydrogens", type='bool', default=True)).process(args=sys.argv[1:]) if len(command_line.args) != 2: command_line.parser.print_help() sys.exit(1) reference_structure = easy_pickle.load(command_line.args[0]) if (type(reference_structure) in (type([]), type(()))): reference_structure = reference_structure[0] structures = easy_pickle.load(command_line.args[1]) if (not type(structures) in (type([]), type(()))): structures = [structures] if not command_line.options.match_hydrogens: reference_structure.select_inplace( ~reference_structure.element_selection('H')) for structure in structures: structure.select_inplace(~structure.element_selection('H')) print "Reference model:" reference_structure.show_summary() print reference_model = reference_structure.as_emma_model() match_list = [] match_histogram = dicts.with_default_value(0) for structure in structures: structure.show_summary() if (hasattr(structure, "info")): print structure.info print sys.stdout.flush() refined_matches = emma.model_matches( reference_model, structure.as_emma_model(), tolerance=command_line.options.tolerance, models_are_diffraction_index_equivalent=False, break_if_match_with_no_singles=True).refined_matches if (len(refined_matches)): refined_matches[0].show() m = len(refined_matches[0].pairs) else: print "No matches" m = 0 match_list.append(match_record(m, structure.scatterers().size())) match_histogram[m] += 1 print sys.stdout.flush() print "match_list:", match_list keys = match_histogram.keys() keys.sort() keys.reverse() print "matches: frequency" sum = 0 for key in keys: v = match_histogram[key] sum += v s = 0 for key in keys: v = match_histogram[key] s += v print " %3d: %3d = %5.1f%%, %5.1f%%" % (key, v, 100. * v / sum, 100. * s / sum) print sys.stdout.flush()
# LIBTBX_SET_DISPATCHER_NAME cxi.xes_finalise # LIBTBX_PRE_DISPATCHER_INCLUDE_SH PHENIX_GUI_ENVIRONMENT=1 # LIBTBX_PRE_DISPATCHER_INCLUDE_SH export PHENIX_GUI_ENVIRONMENT import os import sys from libtbx.option_parser import option_parser from xfel.cxi.cspad_ana.xes_finalise import xes_finalise if (__name__ == "__main__"): args = sys.argv[1:] assert len(args) > 0 command_line = (option_parser() .option("--roi", type="string", help="Region of interest for summing up spectrum.") .option("--output_dirname", "-o", type="string", help="Directory for output files.") ).process(args=args) roi = command_line.options.roi output_dirname = command_line.options.output_dirname runs = command_line.args if output_dirname is None: output_dirname = os.path.join(runs[0], "finalise") print "Output directory: %s" %output_dirname if not len(runs) > 0: print "Usage: cxi.finalise [-o result directory] [data directories as r0xxx/nnn ...]" xes_finalise(runs, output_dirname=output_dirname, roi=roi)
def run(args): assert len(args) > 0 command_line = ( option_parser().option( "--roi", type="string", help="Region of interest for summing up histograms" "from neighbouring pixels.").option( "--log_scale", action="store_true", default=False, help="Draw y-axis on a log scale.").option( "--normalise", action="store_true", default=False, help="Normalise by number of member images.").option( "--save", action="store_true", default=False, help="Save each plot as a png.").option( "--start", type="string", help="Starting pixel coordinates").option( "--fit_gaussians", action="store_true", default=False, help="Fit gaussians to the peaks.").option( "--n_gaussians", type="int", default=2, help="Number of gaussians to fit."). option( "--estimated_gain", type="float", default=30, help="The approximate position of the one photon peak.")).process( args=args) log_scale = command_line.options.log_scale fit_gaussians = command_line.options.fit_gaussians roi = cspad_tbx.getOptROI(command_line.options.roi) normalise = command_line.options.normalise save_image = command_line.options.save starting_pixel = command_line.options.start n_gaussians = command_line.options.n_gaussians estimated_gain = command_line.options.estimated_gain if starting_pixel is not None: starting_pixel = eval(starting_pixel) assert isinstance(starting_pixel, tuple) args = command_line.args path = args[0] window_title = path d = easy_pickle.load(path) args = args[1:] pixels = None if len(args) > 0: pixels = [eval(arg) for arg in args] for pixel in pixels: assert isinstance(pixel, tuple) assert len(pixel) == 2 if roi is not None: summed_hist = {} for i in range(roi[2], roi[3]): for j in range(roi[0], roi[1]): if (i, j) not in summed_hist: summed_hist[(0, 0)] = d[(i, j)] else: summed_hist[(0, 0)].update(d[(i, j)]) d = summed_hist #if roi is not None: #summed_hist = None #for i in range(roi[2], roi[3]): #for j in range(roi[0], roi[1]): #if summed_hist is None: #summed_hist = d[(i,j)] #else: #summed_hist.update(d[(i,j)]) #title = str(roi) #plot(hist, window_title=window_title, title=title,log_scale=log_scale, #normalise=normalise, save_image=save_image, fit_gaussians=fit_gaussians) #return histograms = pixel_histograms(d, estimated_gain=estimated_gain) histograms.plot(pixels=pixels, starting_pixel=starting_pixel, fit_gaussians=fit_gaussians, n_gaussians=n_gaussians, window_title=window_title, log_scale=log_scale, save_image=save_image)