def main(): log = logging.getLogger(__name__) parser = get_cli_parser() args = parser.parse_args() config_path = args.config generate_config = args.generate_config config_overwrite = args.overwrite iqr_state_fp = args.iqr_state is_debug = args.debug initialize_logging(logging.getLogger(), is_debug and logging.DEBUG or logging.INFO) log.debug("Showing debug messages.") config = get_default_config() config_loaded = False if config_path and os.path.isfile(config_path): with open(config_path) as f: log.info("Loading configuration: %s", config_path) config.update( json.load(f) ) config_loaded = True output_config(generate_config, config, log, config_overwrite, 100) if not config_loaded: log.error("No configuration provided") exit(101) if not os.path.isfile(iqr_state_fp): log.error("IQR Session info JSON filepath was invalid") exit(102) train_classifier_iqr(config, iqr_state_fp)
def main(): log = logging.getLogger(__name__) parser = get_cli_parser() args = parser.parse_args() config_path = args.config generate_config = args.generate_config config_overwrite = args.overwrite is_debug = args.debug label = args.label file_globs = args.file_globs initialize_logging(logging.getLogger(__name__), is_debug and logging.DEBUG or logging.INFO) initialize_logging(logging.getLogger('smqtk'), is_debug and logging.DEBUG or logging.INFO) log.debug("Showing debug messages.") config = get_default_config() config_loaded = False if config_path and os.path.isfile(config_path): with open(config_path) as f: log.info("Loading configuration: %s", config_path) config.update(json.load(f)) config_loaded = True output_config(generate_config, config, log, config_overwrite, 100) if not config_loaded: log.error("No configuration provided") exit(101) classify_files(config, label, file_globs)
def main(): parser = cli_parser() args = parser.parse_args() llevel = logging.DEBUG if args.verbose else logging.INFO bin_utils.initialize_logging(logging.getLogger(), llevel) log = logging.getLogger("main") # Merge loaded config with default config = default_config() if args.config: if osp.isfile(args.config): with open(args.config, 'r') as f: config.update(json.load(f)) elif not osp.isfile(args.config): log.error("Configuration file path not valid.") exit(1) bin_utils.output_config(args.output_config, config, log, True) # Default config options for this util are valid for running, so no "has # config loaded check here. port = int(config['port']) authkey = str(config['authkey']) mgr = ProxyManager(('', port), authkey) mgr.get_server().serve_forever()
def main(): log = logging.getLogger(__name__) parser = get_cli_parser() args = parser.parse_args() config_path = args.config generate_config = args.generate_config config_overwrite = args.overwrite iqr_state_fp = args.iqr_state is_debug = args.debug initialize_logging(logging.getLogger(), is_debug and logging.DEBUG or logging.INFO) log.debug("Showing debug messages.") config = get_default_config() config_loaded = False if config_path and os.path.isfile(config_path): with open(config_path) as f: log.info("Loading configuration: %s", config_path) config.update(json.load(f)) config_loaded = True output_config(generate_config, config, log, config_overwrite, 100) if not config_loaded: log.error("No configuration provided") exit(101) if not os.path.isfile(iqr_state_fp): log.error("IQR Session info JSON filepath was invalid") exit(102) train_classifier_iqr(config, iqr_state_fp)
def main(): log = logging.getLogger(__name__) parser = get_cli_parser() args = parser.parse_args() config_path = args.config generate_config = args.generate_config config_overwrite = args.overwrite is_debug = args.verbose label = args.label file_globs = args.file_globs initialize_logging(logging.getLogger('__main__'), is_debug and logging.DEBUG or logging.INFO) initialize_logging(logging.getLogger('smqtk'), is_debug and logging.DEBUG or logging.INFO) log.debug("Showing debug messages.") config = get_default_config() config_loaded = False if config_path and os.path.isfile(config_path): with open(config_path) as f: log.info("Loading configuration: %s", config_path) config.update( json.load(f) ) config_loaded = True output_config(generate_config, config, log, config_overwrite, 100) if not config_loaded: log.error("No configuration provided") exit(101) classify_files(config, label, file_globs)
def main(): parser = cli_parser() args = parser.parse_args() bin_utils.initialize_logging(logging.getLogger(), logging.INFO - (10 * args.verbose)) log = logging.getLogger("main") # Merge loaded config with default config_loaded = False config = default_config() if args.config: if osp.isfile(args.config): with open(args.config, 'r') as f: config.update(json.load(f)) config_loaded = True elif not osp.isfile(args.config): log.error("Configuration file path not valid.") exit(1) # output configuration dictionary when asked for. bin_utils.output_config(args.output_config, config, log, True) if not config_loaded: log.error("No configuration provided") exit(1) log.debug("Script arguments:\n%s" % args) def iter_input_elements(): for f in args.input_files: f = osp.expanduser(f) if osp.isfile(f): yield DataFileElement(f) else: log.debug("Expanding glob: %s" % f) for g in glob.glob(f): yield DataFileElement(g) log.info("Adding elements to data set") #: :type: smqtk.representation.DataSet ds = plugin.from_plugin_config(config['data_set'], get_data_set_impls()) ds.add_data(*iter_input_elements())
def main(): usage = "%prog [options] GLOB [ GLOB [ ... ] ]" description = "Add a set of local system files to a data set via " \ "explicit paths or shell-style glob strings." parser = bin_utils.SMQTKOptParser(usage, description=description) parser.add_option('-c', '--config', help="Path to the JSON configuration file") parser.add_option('--output-config', help="Optional path to output a default configuration " "file to. This output file should be modified and " "used for this executable.") parser.add_option('-v', '--verbose', action='store_true', default=False, help='Add debug messaged to output logging.') opts, args = parser.parse_args() bin_utils.initialize_logging(logging.getLogger(), logging.INFO - (10*opts.verbose)) log = logging.getLogger("main") # output configuration dictionary when asked for. bin_utils.output_config(opts.output_config, default_config(), log) with open(opts.config, 'r') as f: config = json.load(f) #: :type: smqtk.representation.DataSet ds = plugin.from_plugin_config(config['data_set'], get_data_set_impls) log.debug("Script arguments:\n%s" % args) def ingest_file(fp): ds.add_data(DataFileElement(fp)) for f in args: f = osp.expanduser(f) if osp.isfile(f): ingest_file(f) else: log.debug("Expanding glob: %s" % f) for g in glob.glob(f): ingest_file(g)
def main(): parser = cli_parser() args = parser.parse_args() debug_smqtk = args.debug_smqtk debug_server = args.debug_server bin_utils.initialize_logging(logging.getLogger("smqtk"), logging.INFO - (10 * debug_smqtk)) bin_utils.initialize_logging(logging.getLogger("werkzeug"), logging.WARN - (20 * debug_server)) log = logging.getLogger("smqtk.main") web_applications = smqtk.web.get_web_applications() if args.list: log.info("") log.info("Available applications:") log.info("") for l in web_applications: log.info("\t" + l) log.info("") exit(0) application_name = args.application if application_name is None: log.error("No application name given!") exit(1) elif application_name not in web_applications: log.error("Invalid application label '%s'", application_name) exit(1) app_class = web_applications[application_name] # Merge loaded config with default config_loaded = False config = app_class.get_default_config() if args.config: if os.path.isfile(args.config): with open(args.config, 'r') as f: config.update(json.load(f)) config_loaded = True elif not os.path.isfile(args.config): log.error("Configuration file path not valid.") exit(1) # Output config and exit if requested bin_utils.output_config(args.output_config, config, log, args.overwrite) # Configuration must have been loaded at this point since we can't normally # trust the default. if not config_loaded: log.error("No configuration provided") exit(1) host = args.host port = args.port and int(args.port) use_reloader = args.reload use_threading = args.threaded use_basic_auth = args.use_basic_auth # noinspection PyUnresolvedReferences app = app_class.from_config(config) if use_basic_auth: app.config["BASIC_AUTH_FORCE"] = True BasicAuth(app) app.config['DEBUG'] = debug_server app.run(host=host, port=port, debug=debug_server, use_reloader=use_reloader, threaded=use_threading)
def main(): usage = "%prog [OPTIONS] INPUT_FILE" description = """\ Compute a descriptor vector for a given data file, outputting the generated feature vector to standard out, or to an output file if one was specified (in numpy format). """ parser = bin_utils.SMQTKOptParser(usage, description=description) group_labels = optparse.OptionGroup(parser, "Configuration") group_labels.add_option('-c', '--config', default=None, help='Path to the JSON configuration file.') group_labels.add_option('--output-config', default=None, help='Optional path to output default JSON ' 'configuration to.') parser.add_option_group(group_labels) group_optional = optparse.OptionGroup(parser, "Optional Parameters") group_optional.add_option('--overwrite', action='store_true', default=False, help="Force descriptor computation even if an " "existing descriptor vector was discovered " "based on the given content descriptor type " "and data combination.") group_optional.add_option('-o', '--output-filepath', help='Optional path to a file to output feature ' 'vector to. Otherwise the feature vector is ' 'printed to standard out. Output is saved ' 'in numpy binary format (.npy suffix ' 'recommended).') group_optional.add_option('-v', '--verbose', action='store_true', default=False, help='Print additional debugging messages. All ' 'logging goes to standard error.') parser.add_option_group(group_optional) opts, args = parser.parse_args() output_filepath = opts.output_filepath overwrite = opts.overwrite verbose = opts.verbose llevel = logging.DEBUG if verbose else logging.INFO bin_utils.initialize_logging(logging.getLogger(), llevel) log = logging.getLogger("main") bin_utils.output_config(opts.output_config, default_config(), log) if not opts.config: log.error("No configuration provided") exit(1) elif not os.path.isfile(opts.config): log.error("Configuration file path not valid.") exit(1) if len(args) == 0: log.error("Failed to provide an input file path") exit(1) if len(args) > 1: log.warning("More than one filepath provided as an argument. Only " "computing for the first one.") with open(opts.config, 'r') as f: config = json.load(f) input_filepath = args[0] data_element = DataFileElement(input_filepath) factory = DescriptorElementFactory.from_config(config['descriptor_factory']) #: :type: smqtk.descriptor_generator.DescriptorGenerator cd = plugin.from_plugin_config(config['content_descriptor'], get_descriptor_generator_impls) descr_elem = cd.compute_descriptor(data_element, factory, overwrite) vec = descr_elem.vector() if vec is None: log.error("Failed to generate a descriptor vector for the input data!") if output_filepath: numpy.save(output_filepath, vec) else: # Construct string, because numpy s = [] # noinspection PyTypeChecker for f in vec: s.append('%15f' % f) print ' '.join(s)
l = logging.getLogger(__name__) # Merge loaded config with default config_loaded = False c = default_config() if config_fp: if os.path.isfile(config_fp): with open(config_fp) as f: c.update(json.loads(jsmin(f.read()))) config_loaded = True else: l.error("Config file path not valid") exit(100) output_config(out_config_fp, c, overwrite=True) # Input checking if not config_loaded: l.error("No configuration provided") exit(101) if not filelist_fp: l.error("No file-list file specified") exit(102) elif not os.path.isfile(filelist_fp): l.error("Invalid file list path: %s", filelist_fp) exit(103) if not completed_files_fp: l.error("No complete files output specified")
def main(): parser = cli_parser() args = parser.parse_args() output_filepath = args.output_filepath overwrite = args.overwrite verbose = args.verbose llevel = logging.DEBUG if verbose else logging.INFO bin_utils.initialize_logging(logging.getLogger(), llevel) log = logging.getLogger("main") # Merge loaded config with default config_loaded = False config = default_config() if args.config: if os.path.isfile(args.config): with open(args.config, 'r') as f: config.update(json.load(f)) config_loaded = True elif not os.path.isfile(args.config): log.error("Configuration file path not valid.") exit(1) bin_utils.output_config(args.output_config, config, log, True) # Configuration must have been loaded at this point since we can't normally # trust the default. if not config_loaded: log.error("No configuration provided") exit(1) if not args.input_file: log.error("Failed to provide an input file path") exit(1) elif not os.path.isfile(args.input_file): log.error("Given path does not point to a file.") exit(1) input_filepath = args.input_file data_element = DataFileElement(input_filepath) factory = DescriptorElementFactory.from_config( config['descriptor_factory']) #: :type: smqtk.algorithms.descriptor_generator.DescriptorGenerator cd = plugin.from_plugin_config(config['content_descriptor'], get_descriptor_generator_impls) descr_elem = cd.compute_descriptor(data_element, factory, overwrite) vec = descr_elem.vector() if vec is None: log.error("Failed to generate a descriptor vector for the input data!") if output_filepath: numpy.save(output_filepath, vec) else: # Construct string, because numpy s = [] # noinspection PyTypeChecker for f in vec: s.append('%15f' % f) print ' '.join(s)
def main(): parser = bin_utils.SMQTKOptParser() setup_cli(parser) opts, args = parser.parse_args() debug_smqtk = opts.debug_smqtk debug_server = opts.debug_server bin_utils.initialize_logging(logging.getLogger("smqtk"), logging.INFO - (10*debug_smqtk)) bin_utils.initialize_logging(logging.getLogger("werkzeug"), logging.WARN - (20*debug_server)) log = logging.getLogger("smqtk.main") web_applications = smqtk.web.get_web_applications() if opts.list: log.info("") log.info("Available applications:") log.info("") for l in web_applications: log.info("\t" + l) log.info("") exit(0) application_name = opts.application if application_name is None: log.error("No application name given!") exit(1) elif application_name not in web_applications: log.error("Invalid application label '%s'", application_name) exit(1) app_class = web_applications[application_name] # Output config and exit if requested bin_utils.output_config(opts.output_config, app_class.get_default_config(), log, opts.overwrite) if not opts.config: log.error("No configuration provided") exit(1) elif not os.path.isfile(opts.config): log.error("Configuration file path not valid.") exit(1) with open(opts.config, 'r') as f: config = json.loads(jsmin(f.read())) host = opts.host port = opts.port and int(opts.port) use_reloader = opts.reload use_threading = opts.threaded use_basic_auth = opts.use_basic_auth # noinspection PyUnresolvedReferences app = app_class.from_config(config) if use_basic_auth: app.config["BASIC_AUTH_FORCE"] = True BasicAuth(app) app.config['DEBUG'] = debug_server app.run(host=host, port=port, debug=debug_server, use_reloader=use_reloader, threaded=use_threading)
def main(): parser = cli_parser() args = parser.parse_args() debug_smqtk = args.debug_smqtk debug_server = args.debug_server bin_utils.initialize_logging(logging.getLogger("smqtk"), logging.INFO - (10*debug_smqtk)) bin_utils.initialize_logging(logging.getLogger("werkzeug"), logging.WARN - (20*debug_server)) log = logging.getLogger("smqtk.main") web_applications = smqtk.web.get_web_applications() if args.list: log.info("") log.info("Available applications:") log.info("") for l in web_applications: log.info("\t" + l) log.info("") exit(0) application_name = args.application if application_name is None: log.error("No application name given!") exit(1) elif application_name not in web_applications: log.error("Invalid application label '%s'", application_name) exit(1) app_class = web_applications[application_name] # Merge loaded config with default config_loaded = False config = app_class.get_default_config() if args.config: if os.path.isfile(args.config): with open(args.config, 'r') as f: config.update(json.load(f)) config_loaded = True elif not os.path.isfile(args.config): log.error("Configuration file path not valid.") exit(1) # Output config and exit if requested bin_utils.output_config(args.output_config, config, log, args.overwrite) # Configuration must have been loaded at this point since we can't normally # trust the default. if not config_loaded: log.error("No configuration provided") exit(1) host = args.host port = args.port and int(args.port) use_reloader = args.reload use_threading = args.threaded use_basic_auth = args.use_basic_auth # noinspection PyUnresolvedReferences app = app_class.from_config(config) if use_basic_auth: app.config["BASIC_AUTH_FORCE"] = True BasicAuth(app) app.config['DEBUG'] = debug_server app.run(host=host, port=port, debug=debug_server, use_reloader=use_reloader, threaded=use_threading)
def main(): parser = cli_parser() args = parser.parse_args() output_filepath = args.output_filepath overwrite = args.overwrite verbose = args.verbose llevel = logging.DEBUG if verbose else logging.INFO bin_utils.initialize_logging(logging.getLogger(), llevel) log = logging.getLogger("main") # Merge loaded config with default config_loaded = False config = default_config() if args.config: if os.path.isfile(args.config): with open(args.config, 'r') as f: config.update(json.load(f)) config_loaded = True elif not os.path.isfile(args.config): log.error("Configuration file path not valid.") exit(1) bin_utils.output_config(args.output_config, config, log, True) # Configuration must have been loaded at this point since we can't normally # trust the default. if not config_loaded: log.error("No configuration provided") exit(1) if not args.input_file: log.error("Failed to provide an input file path") exit(1) elif not os.path.isfile(args.input_file): log.error("Given path does not point to a file.") exit(1) input_filepath = args.input_file data_element = DataFileElement(input_filepath) factory = DescriptorElementFactory.from_config(config['descriptor_factory']) #: :type: smqtk.algorithms.descriptor_generator.DescriptorGenerator cd = plugin.from_plugin_config(config['content_descriptor'], get_descriptor_generator_impls()) descr_elem = cd.compute_descriptor(data_element, factory, overwrite) vec = descr_elem.vector() if vec is None: log.error("Failed to generate a descriptor vector for the input data!") if output_filepath: numpy.save(output_filepath, vec) else: # Construct string, because numpy s = [] # noinspection PyTypeChecker for f in vec: s.append('%15f' % f) print ' '.join(s)