def main():
log = logging.getLogger(__name__)
parser = get_cli_parser()
args = parser.parse_args()
config_path = args.config
generate_config = args.generate_config
config_overwrite = args.overwrite
iqr_state_fp = args.iqr_state
is_debug = args.debug
initialize_logging(logging.getLogger(),
is_debug and logging.DEBUG or logging.INFO)
log.debug("Showing debug messages.")
config = get_default_config()
config_loaded = False
if config_path and os.path.isfile(config_path):
with open(config_path) as f:
log.info("Loading configuration: %s", config_path)
config.update(
json.load(f)
)
config_loaded = True
output_config(generate_config, config, log, config_overwrite, 100)
if not config_loaded:
log.error("No configuration provided")
exit(101)
if not os.path.isfile(iqr_state_fp):
log.error("IQR Session info JSON filepath was invalid")
exit(102)
train_classifier_iqr(config, iqr_state_fp)
def main():
log = logging.getLogger(__name__)
parser = get_cli_parser()
args = parser.parse_args()
config_path = args.config
generate_config = args.generate_config
config_overwrite = args.overwrite
is_debug = args.debug
label = args.label
file_globs = args.file_globs
initialize_logging(logging.getLogger(__name__), is_debug and logging.DEBUG
or logging.INFO)
initialize_logging(logging.getLogger('smqtk'), is_debug and logging.DEBUG
or logging.INFO)
log.debug("Showing debug messages.")
config = get_default_config()
config_loaded = False
if config_path and os.path.isfile(config_path):
with open(config_path) as f:
log.info("Loading configuration: %s", config_path)
config.update(json.load(f))
config_loaded = True
output_config(generate_config, config, log, config_overwrite, 100)
if not config_loaded:
log.error("No configuration provided")
exit(101)
classify_files(config, label, file_globs)
def main():
parser = cli_parser()
args = parser.parse_args()
llevel = logging.DEBUG if args.verbose else logging.INFO
bin_utils.initialize_logging(logging.getLogger(), llevel)
log = logging.getLogger("main")
# Merge loaded config with default
config = default_config()
if args.config:
if osp.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
elif not osp.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
bin_utils.output_config(args.output_config, config, log, True)
# Default config options for this util are valid for running, so no "has
# config loaded check here.
port = int(config['port'])
authkey = str(config['authkey'])
mgr = ProxyManager(('', port), authkey)
mgr.get_server().serve_forever()
def main():
log = logging.getLogger(__name__)
parser = get_cli_parser()
args = parser.parse_args()
config_path = args.config
generate_config = args.generate_config
config_overwrite = args.overwrite
iqr_state_fp = args.iqr_state
is_debug = args.debug
initialize_logging(logging.getLogger(), is_debug and logging.DEBUG
or logging.INFO)
log.debug("Showing debug messages.")
config = get_default_config()
config_loaded = False
if config_path and os.path.isfile(config_path):
with open(config_path) as f:
log.info("Loading configuration: %s", config_path)
config.update(json.load(f))
config_loaded = True
output_config(generate_config, config, log, config_overwrite, 100)
if not config_loaded:
log.error("No configuration provided")
exit(101)
if not os.path.isfile(iqr_state_fp):
log.error("IQR Session info JSON filepath was invalid")
exit(102)
train_classifier_iqr(config, iqr_state_fp)
def main():
parser = cli_parser()
args = parser.parse_args()
llevel = logging.DEBUG if args.verbose else logging.INFO
bin_utils.initialize_logging(logging.getLogger(), llevel)
log = logging.getLogger("main")
# Merge loaded config with default
config = default_config()
if args.config:
if osp.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
elif not osp.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
bin_utils.output_config(args.output_config, config, log, True)
# Default config options for this util are valid for running, so no "has
# config loaded check here.
port = int(config['port'])
authkey = str(config['authkey'])
mgr = ProxyManager(('', port), authkey)
mgr.get_server().serve_forever()
def main():
log = logging.getLogger(__name__)
parser = get_cli_parser()
args = parser.parse_args()
config_path = args.config
generate_config = args.generate_config
config_overwrite = args.overwrite
is_debug = args.verbose
label = args.label
file_globs = args.file_globs
initialize_logging(logging.getLogger('__main__'),
is_debug and logging.DEBUG or logging.INFO)
initialize_logging(logging.getLogger('smqtk'),
is_debug and logging.DEBUG or logging.INFO)
log.debug("Showing debug messages.")
config = get_default_config()
config_loaded = False
if config_path and os.path.isfile(config_path):
with open(config_path) as f:
log.info("Loading configuration: %s", config_path)
config.update(
json.load(f)
)
config_loaded = True
output_config(generate_config, config, log, config_overwrite, 100)
if not config_loaded:
log.error("No configuration provided")
exit(101)
classify_files(config, label, file_globs)
def main():
parser = cli_parser()
args = parser.parse_args()
bin_utils.initialize_logging(logging.getLogger(),
logging.INFO - (10 * args.verbose))
log = logging.getLogger("main")
# Merge loaded config with default
config_loaded = False
config = default_config()
if args.config:
if osp.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
config_loaded = True
elif not osp.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
# output configuration dictionary when asked for.
bin_utils.output_config(args.output_config, config, log, True)
if not config_loaded:
log.error("No configuration provided")
exit(1)
log.debug("Script arguments:\n%s" % args)
def iter_input_elements():
for f in args.input_files:
f = osp.expanduser(f)
if osp.isfile(f):
yield DataFileElement(f)
else:
log.debug("Expanding glob: %s" % f)
for g in glob.glob(f):
yield DataFileElement(g)
log.info("Adding elements to data set")
#: :type: smqtk.representation.DataSet
ds = plugin.from_plugin_config(config['data_set'], get_data_set_impls())
ds.add_data(*iter_input_elements())
def main():
parser = cli_parser()
args = parser.parse_args()
bin_utils.initialize_logging(logging.getLogger(),
logging.INFO - (10 * args.verbose))
log = logging.getLogger("main")
# Merge loaded config with default
config_loaded = False
config = default_config()
if args.config:
if osp.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
config_loaded = True
elif not osp.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
# output configuration dictionary when asked for.
bin_utils.output_config(args.output_config, config, log, True)
if not config_loaded:
log.error("No configuration provided")
exit(1)
log.debug("Script arguments:\n%s" % args)
def iter_input_elements():
for f in args.input_files:
f = osp.expanduser(f)
if osp.isfile(f):
yield DataFileElement(f)
else:
log.debug("Expanding glob: %s" % f)
for g in glob.glob(f):
yield DataFileElement(g)
log.info("Adding elements to data set")
#: :type: smqtk.representation.DataSet
ds = plugin.from_plugin_config(config['data_set'], get_data_set_impls())
ds.add_data(*iter_input_elements())
def main():
usage = "%prog [options] GLOB [ GLOB [ ... ] ]"
description = "Add a set of local system files to a data set via " \
"explicit paths or shell-style glob strings."
parser = bin_utils.SMQTKOptParser(usage, description=description)
parser.add_option('-c', '--config',
help="Path to the JSON configuration file")
parser.add_option('--output-config',
help="Optional path to output a default configuration "
"file to. This output file should be modified and "
"used for this executable.")
parser.add_option('-v', '--verbose', action='store_true', default=False,
help='Add debug messaged to output logging.')
opts, args = parser.parse_args()
bin_utils.initialize_logging(logging.getLogger(),
logging.INFO - (10*opts.verbose))
log = logging.getLogger("main")
# output configuration dictionary when asked for.
bin_utils.output_config(opts.output_config, default_config(), log)
with open(opts.config, 'r') as f:
config = json.load(f)
#: :type: smqtk.representation.DataSet
ds = plugin.from_plugin_config(config['data_set'], get_data_set_impls)
log.debug("Script arguments:\n%s" % args)
def ingest_file(fp):
ds.add_data(DataFileElement(fp))
for f in args:
f = osp.expanduser(f)
if osp.isfile(f):
ingest_file(f)
else:
log.debug("Expanding glob: %s" % f)
for g in glob.glob(f):
ingest_file(g)
def main():
parser = cli_parser()
args = parser.parse_args()
debug_smqtk = args.debug_smqtk
debug_server = args.debug_server
bin_utils.initialize_logging(logging.getLogger("smqtk"),
logging.INFO - (10 * debug_smqtk))
bin_utils.initialize_logging(logging.getLogger("werkzeug"),
logging.WARN - (20 * debug_server))
log = logging.getLogger("smqtk.main")
web_applications = smqtk.web.get_web_applications()
if args.list:
log.info("")
log.info("Available applications:")
log.info("")
for l in web_applications:
log.info("\t" + l)
log.info("")
exit(0)
application_name = args.application
if application_name is None:
log.error("No application name given!")
exit(1)
elif application_name not in web_applications:
log.error("Invalid application label '%s'", application_name)
exit(1)
app_class = web_applications[application_name]
# Merge loaded config with default
config_loaded = False
config = app_class.get_default_config()
if args.config:
if os.path.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
config_loaded = True
elif not os.path.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
# Output config and exit if requested
bin_utils.output_config(args.output_config, config, log, args.overwrite)
# Configuration must have been loaded at this point since we can't normally
# trust the default.
if not config_loaded:
log.error("No configuration provided")
exit(1)
host = args.host
port = args.port and int(args.port)
use_reloader = args.reload
use_threading = args.threaded
use_basic_auth = args.use_basic_auth
# noinspection PyUnresolvedReferences
app = app_class.from_config(config)
if use_basic_auth:
app.config["BASIC_AUTH_FORCE"] = True
BasicAuth(app)
app.config['DEBUG'] = debug_server
app.run(host=host,
port=port,
debug=debug_server,
use_reloader=use_reloader,
threaded=use_threading)
def main():
usage = "%prog [OPTIONS] INPUT_FILE"
description = """\
Compute a descriptor vector for a given data file, outputting the generated
feature vector to standard out, or to an output file if one was specified (in
numpy format).
"""
parser = bin_utils.SMQTKOptParser(usage, description=description)
group_labels = optparse.OptionGroup(parser, "Configuration")
group_labels.add_option('-c', '--config',
default=None,
help='Path to the JSON configuration file.')
group_labels.add_option('--output-config',
default=None,
help='Optional path to output default JSON '
'configuration to.')
parser.add_option_group(group_labels)
group_optional = optparse.OptionGroup(parser, "Optional Parameters")
group_optional.add_option('--overwrite',
action='store_true', default=False,
help="Force descriptor computation even if an "
"existing descriptor vector was discovered "
"based on the given content descriptor type "
"and data combination.")
group_optional.add_option('-o', '--output-filepath',
help='Optional path to a file to output feature '
'vector to. Otherwise the feature vector is '
'printed to standard out. Output is saved '
'in numpy binary format (.npy suffix '
'recommended).')
group_optional.add_option('-v', '--verbose',
action='store_true', default=False,
help='Print additional debugging messages. All '
'logging goes to standard error.')
parser.add_option_group(group_optional)
opts, args = parser.parse_args()
output_filepath = opts.output_filepath
overwrite = opts.overwrite
verbose = opts.verbose
llevel = logging.DEBUG if verbose else logging.INFO
bin_utils.initialize_logging(logging.getLogger(), llevel)
log = logging.getLogger("main")
bin_utils.output_config(opts.output_config, default_config(), log)
if not opts.config:
log.error("No configuration provided")
exit(1)
elif not os.path.isfile(opts.config):
log.error("Configuration file path not valid.")
exit(1)
if len(args) == 0:
log.error("Failed to provide an input file path")
exit(1)
if len(args) > 1:
log.warning("More than one filepath provided as an argument. Only "
"computing for the first one.")
with open(opts.config, 'r') as f:
config = json.load(f)
input_filepath = args[0]
data_element = DataFileElement(input_filepath)
factory = DescriptorElementFactory.from_config(config['descriptor_factory'])
#: :type: smqtk.descriptor_generator.DescriptorGenerator
cd = plugin.from_plugin_config(config['content_descriptor'],
get_descriptor_generator_impls)
descr_elem = cd.compute_descriptor(data_element, factory, overwrite)
vec = descr_elem.vector()
if vec is None:
log.error("Failed to generate a descriptor vector for the input data!")
if output_filepath:
numpy.save(output_filepath, vec)
else:
# Construct string, because numpy
s = []
# noinspection PyTypeChecker
for f in vec:
s.append('%15f' % f)
print ' '.join(s)
l = logging.getLogger(__name__)
# Merge loaded config with default
config_loaded = False
c = default_config()
if config_fp:
if os.path.isfile(config_fp):
with open(config_fp) as f:
c.update(json.loads(jsmin(f.read())))
config_loaded = True
else:
l.error("Config file path not valid")
exit(100)
output_config(out_config_fp, c, overwrite=True)
# Input checking
if not config_loaded:
l.error("No configuration provided")
exit(101)
if not filelist_fp:
l.error("No file-list file specified")
exit(102)
elif not os.path.isfile(filelist_fp):
l.error("Invalid file list path: %s", filelist_fp)
exit(103)
if not completed_files_fp:
l.error("No complete files output specified")
def main():
parser = cli_parser()
args = parser.parse_args()
output_filepath = args.output_filepath
overwrite = args.overwrite
verbose = args.verbose
llevel = logging.DEBUG if verbose else logging.INFO
bin_utils.initialize_logging(logging.getLogger(), llevel)
log = logging.getLogger("main")
# Merge loaded config with default
config_loaded = False
config = default_config()
if args.config:
if os.path.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
config_loaded = True
elif not os.path.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
bin_utils.output_config(args.output_config, config, log, True)
# Configuration must have been loaded at this point since we can't normally
# trust the default.
if not config_loaded:
log.error("No configuration provided")
exit(1)
if not args.input_file:
log.error("Failed to provide an input file path")
exit(1)
elif not os.path.isfile(args.input_file):
log.error("Given path does not point to a file.")
exit(1)
input_filepath = args.input_file
data_element = DataFileElement(input_filepath)
factory = DescriptorElementFactory.from_config(
config['descriptor_factory'])
#: :type: smqtk.algorithms.descriptor_generator.DescriptorGenerator
cd = plugin.from_plugin_config(config['content_descriptor'],
get_descriptor_generator_impls)
descr_elem = cd.compute_descriptor(data_element, factory, overwrite)
vec = descr_elem.vector()
if vec is None:
log.error("Failed to generate a descriptor vector for the input data!")
if output_filepath:
numpy.save(output_filepath, vec)
else:
# Construct string, because numpy
s = []
# noinspection PyTypeChecker
for f in vec:
s.append('%15f' % f)
print ' '.join(s)
def main():
parser = bin_utils.SMQTKOptParser()
setup_cli(parser)
opts, args = parser.parse_args()
debug_smqtk = opts.debug_smqtk
debug_server = opts.debug_server
bin_utils.initialize_logging(logging.getLogger("smqtk"),
logging.INFO - (10*debug_smqtk))
bin_utils.initialize_logging(logging.getLogger("werkzeug"),
logging.WARN - (20*debug_server))
log = logging.getLogger("smqtk.main")
web_applications = smqtk.web.get_web_applications()
if opts.list:
log.info("")
log.info("Available applications:")
log.info("")
for l in web_applications:
log.info("\t" + l)
log.info("")
exit(0)
application_name = opts.application
if application_name is None:
log.error("No application name given!")
exit(1)
elif application_name not in web_applications:
log.error("Invalid application label '%s'", application_name)
exit(1)
app_class = web_applications[application_name]
# Output config and exit if requested
bin_utils.output_config(opts.output_config, app_class.get_default_config(),
log, opts.overwrite)
if not opts.config:
log.error("No configuration provided")
exit(1)
elif not os.path.isfile(opts.config):
log.error("Configuration file path not valid.")
exit(1)
with open(opts.config, 'r') as f:
config = json.loads(jsmin(f.read()))
host = opts.host
port = opts.port and int(opts.port)
use_reloader = opts.reload
use_threading = opts.threaded
use_basic_auth = opts.use_basic_auth
# noinspection PyUnresolvedReferences
app = app_class.from_config(config)
if use_basic_auth:
app.config["BASIC_AUTH_FORCE"] = True
BasicAuth(app)
app.config['DEBUG'] = debug_server
app.run(host=host, port=port, debug=debug_server, use_reloader=use_reloader,
threaded=use_threading)
def main():
parser = cli_parser()
args = parser.parse_args()
debug_smqtk = args.debug_smqtk
debug_server = args.debug_server
bin_utils.initialize_logging(logging.getLogger("smqtk"),
logging.INFO - (10*debug_smqtk))
bin_utils.initialize_logging(logging.getLogger("werkzeug"),
logging.WARN - (20*debug_server))
log = logging.getLogger("smqtk.main")
web_applications = smqtk.web.get_web_applications()
if args.list:
log.info("")
log.info("Available applications:")
log.info("")
for l in web_applications:
log.info("\t" + l)
log.info("")
exit(0)
application_name = args.application
if application_name is None:
log.error("No application name given!")
exit(1)
elif application_name not in web_applications:
log.error("Invalid application label '%s'", application_name)
exit(1)
app_class = web_applications[application_name]
# Merge loaded config with default
config_loaded = False
config = app_class.get_default_config()
if args.config:
if os.path.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
config_loaded = True
elif not os.path.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
# Output config and exit if requested
bin_utils.output_config(args.output_config, config, log, args.overwrite)
# Configuration must have been loaded at this point since we can't normally
# trust the default.
if not config_loaded:
log.error("No configuration provided")
exit(1)
host = args.host
port = args.port and int(args.port)
use_reloader = args.reload
use_threading = args.threaded
use_basic_auth = args.use_basic_auth
# noinspection PyUnresolvedReferences
app = app_class.from_config(config)
if use_basic_auth:
app.config["BASIC_AUTH_FORCE"] = True
BasicAuth(app)
app.config['DEBUG'] = debug_server
app.run(host=host, port=port, debug=debug_server, use_reloader=use_reloader,
threaded=use_threading)
def main():
parser = cli_parser()
args = parser.parse_args()
output_filepath = args.output_filepath
overwrite = args.overwrite
verbose = args.verbose
llevel = logging.DEBUG if verbose else logging.INFO
bin_utils.initialize_logging(logging.getLogger(), llevel)
log = logging.getLogger("main")
# Merge loaded config with default
config_loaded = False
config = default_config()
if args.config:
if os.path.isfile(args.config):
with open(args.config, 'r') as f:
config.update(json.load(f))
config_loaded = True
elif not os.path.isfile(args.config):
log.error("Configuration file path not valid.")
exit(1)
bin_utils.output_config(args.output_config, config, log, True)
# Configuration must have been loaded at this point since we can't normally
# trust the default.
if not config_loaded:
log.error("No configuration provided")
exit(1)
if not args.input_file:
log.error("Failed to provide an input file path")
exit(1)
elif not os.path.isfile(args.input_file):
log.error("Given path does not point to a file.")
exit(1)
input_filepath = args.input_file
data_element = DataFileElement(input_filepath)
factory = DescriptorElementFactory.from_config(config['descriptor_factory'])
#: :type: smqtk.algorithms.descriptor_generator.DescriptorGenerator
cd = plugin.from_plugin_config(config['content_descriptor'],
get_descriptor_generator_impls())
descr_elem = cd.compute_descriptor(data_element, factory, overwrite)
vec = descr_elem.vector()
if vec is None:
log.error("Failed to generate a descriptor vector for the input data!")
if output_filepath:
numpy.save(output_filepath, vec)
else:
# Construct string, because numpy
s = []
# noinspection PyTypeChecker
for f in vec:
s.append('%15f' % f)
print ' '.join(s)
l = logging.getLogger(__name__)
# Merge loaded config with default
config_loaded = False
c = default_config()
if config_fp:
if os.path.isfile(config_fp):
with open(config_fp) as f:
c.update(json.loads(jsmin(f.read())))
config_loaded = True
else:
l.error("Config file path not valid")
exit(100)
output_config(out_config_fp, c, overwrite=True)
# Input checking
if not config_loaded:
l.error("No configuration provided")
exit(101)
if not filelist_fp:
l.error("No file-list file specified")
exit(102)
elif not os.path.isfile(filelist_fp):
l.error("Invalid file list path: %s", filelist_fp)
exit(103)
if not completed_files_fp:
l.error("No complete files output specified")
