def main(args=sys.argv[1:]):
"""Main process
"""
if args:
if not args[0].lower() in SUBCOMMANDS:
new_args = ["main"]
new_args.extend(args)
else:
new_args = args
# checks if the old --delete syntax is used
new_args = check_delete_option(new_args)
if not PYTHON3:
new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args]
if new_args[0] == "main":
main_dispatcher(args=new_args)
elif new_args[0] == "analyze":
analyze_dispatcher(args=new_args)
elif new_args[0] == "cluster":
cluster_dispatcher(args=new_args)
elif new_args[0] == "anomaly":
anomaly_dispatcher(args=new_args)
elif new_args[0] == "sample":
sample_dispatcher(args=new_args)
elif new_args[0] == "report":
report_dispatcher(args=new_args)
elif new_args[0] == "reify":
reify_dispatcher(args=new_args)
elif new_args[0] == "execute":
execute_dispatcher(args=new_args)
elif new_args[0] == "delete":
delete_dispatcher(args=new_args)
elif new_args[0] == "project":
project_dispatcher(args=new_args)
elif new_args[0] == "association":
association_dispatcher(args=new_args)
elif new_args[0] == "logistic-regression":
logistic_regression_dispatcher(args=new_args)
elif new_args[0] == "whizzml":
whizzml_dispatcher(args=new_args)
else:
sys.exit(
"BigMLer used with no arguments. Check:\nbigmler --help\n\nor"
"\n\nbigmler sample --help\n\n"
"\n\nbigmler analyze --help\n\n"
"\n\nbigmler cluster --help\n\n"
"\n\nbigmler anomaly --help\n\n"
"\n\nbigmler report --help\n\n"
"\n\nbigmler reify --help\n\n"
"\n\nbigmler project --help\n\n"
"\n\nbigmler association --help\n\n"
"\n\nbigmler logistic-regression --help\n\n"
"\n\nbigmler execute --help\n\n"
"\n\nbigmler whizzml --help\n\n"
"\n\nbigmler delete --help\n\n"
" for a list of options"
)
def main(args=sys.argv[1:]):
"""Main process
"""
if args:
if not args[0].lower() in SUBCOMMANDS:
new_args = ["main"]
new_args.extend(args)
else:
new_args = args
# checks if the old --delete syntax is used
new_args = check_delete_option(new_args)
if not PYTHON3:
new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args]
if new_args[0] == "main":
main_dispatcher(args=new_args)
elif new_args[0] == "analyze":
analyze_dispatcher(args=new_args)
elif new_args[0] == "cluster":
cluster_dispatcher(args=new_args)
elif new_args[0] == "anomaly":
anomaly_dispatcher(args=new_args)
elif new_args[0] == "sample":
sample_dispatcher(args=new_args)
elif new_args[0] == "report":
report_dispatcher(args=new_args)
elif new_args[0] == "reify":
reify_dispatcher(args=new_args)
elif new_args[0] == "execute":
execute_dispatcher(args=new_args)
elif new_args[0] == "delete":
delete_dispatcher(args=new_args)
elif new_args[0] == "project":
project_dispatcher(args=new_args)
elif new_args[0] == "association":
association_dispatcher(args=new_args)
elif new_args[0] == "logistic-regression":
logistic_regression_dispatcher(args=new_args)
elif new_args[0] == "whizzml":
whizzml_dispatcher(args=new_args)
else:
sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor"
"\n\nbigmler sample --help\n\n"
"\n\nbigmler analyze --help\n\n"
"\n\nbigmler cluster --help\n\n"
"\n\nbigmler anomaly --help\n\n"
"\n\nbigmler report --help\n\n"
"\n\nbigmler reify --help\n\n"
"\n\nbigmler project --help\n\n"
"\n\nbigmler association --help\n\n"
"\n\nbigmler logistic-regression --help\n\n"
"\n\nbigmler execute --help\n\n"
"\n\nbigmler whizzml --help\n\n"
"\n\nbigmler delete --help\n\n"
" for a list of options")
def create_package(args, api, common_options, resume=False):
"""Creates the package whizzml resources as referred in the metadata.json
file.
"""
set_subcommand_file(args.output_dir)
if resume:
retrieve_subcommands()
# read the metadata.json information
message = ('Reading the metadata.json files.........\n')
u.log_message(message, log_file=session_file,
console=args.verbosity)
package_dir = args.package_dir
output_dir = args.output_dir
metadata_file = os.path.join(package_dir, METADATA_FILE)
metadata = None
created_resources = []
with open(metadata_file) as metadata_handler:
metadata = json.load(metadata_handler)
# recurse into components/directories, if any
if metadata.get("kind") == "package" and 'components' in metadata:
components = metadata.get("components")
for component in components:
message = ('Inspecting component %s.........\n' % component)
u.log_message(message, log_file=session_file,
console=args.verbosity)
args.package_dir = os.path.join(package_dir, component)
create_package(args, api, common_options, resume=resume)
args.package_dir = package_dir
else:
# create libraries or scripts
imports = []
if metadata.get("imports") is not None:
lib_imports = metadata.get("imports")
for lib_import in lib_imports:
args.package_dir = os.path.join(package_dir, lib_import)
create_package(args, api, common_options, resume=resume)
imports.append(read_library_id(os.path.join( \
output_dir, os.path.basename(args.package_dir))))
args.package_dir = package_dir
# read the metadata.json information
message = ('Creating the %s.........\n' % metadata.get("kind"))
u.log_message(message, log_file=session_file,
console=args.verbosity)
if metadata.get("kind") in WHIZZML_RESOURCES:
whizzml_code = os.path.join(args.package_dir, \
metadata.get("source_code", "%s.whizzml" % \
metadata.get("kind")))
args.output_dir = os.path.join(output_dir, \
os.path.basename(package_dir))
# creating command to create the resource
command = COMMANDS[metadata.get("kind")] % (whizzml_code,
args.output_dir)
command_args = command.split()
bigml.util.check_dir(args.output_dir)
# getting inputs and outputs for the script from metadata
if "inputs" in metadata:
inputs_file = os.path.join(args.output_dir, "inputs.json")
with open(inputs_file, "w") as inputs_handler:
json.dump(metadata.get("inputs"), inputs_handler)
command_args.extend(["--declare-inputs", inputs_file])
if "outputs" in metadata:
outputs_file = os.path.join(args.output_dir, "outputs.json")
with open(outputs_file, "w") as outputs_handler:
json.dump(metadata.get("outputs"), outputs_handler)
command_args.extend(["--declare-outputs", outputs_file])
if "description" in metadata:
desc_file = os.path.join(args.output_dir, "description.txt")
with open(desc_file, "w") as desc_handler:
desc_handler.write(metadata.get("description"))
command_args.extend(["--description", desc_file])
if metadata.get("name"):
command_args.extend(["--name", metadata.get("name")])
# adding imports, if any
if imports:
command_args.extend(["--imports", ",".join(imports)])
command_args.extend(["--verbosity", str(args.verbosity)])
if resume:
next_command = subcommand_list.pop()
if different_command(next_command, command):
resume = False
u.sys_log_message(command, log_file=subcommand_file)
execute_dispatcher(args=command_args)
elif not subcommand_list:
execute_dispatcher(args=['execute', '--resume'])
resume = False
else:
u.sys_log_message(command, log_file=subcommand_file)
execute_dispatcher(args=command_args)
args.output_dir = output_dir
def create_package(args, api, command_obj, resume=False):
"""Creates the package whizzml resources as referred in the metadata.json
file.
"""
set_subcommand_file(args.output_dir)
if resume:
retrieve_subcommands()
# read the metadata.json information
message = ('Reading the metadata.json files.........\n')
u.log_message(message, log_file=session_file, console=args.verbosity)
package_dir = args.package_dir
output_dir = args.output_dir
metadata_file = os.path.join(package_dir, METADATA_FILE)
metadata = None
with open(metadata_file) as metadata_handler:
metadata = json.load(metadata_handler)
# recurse into components/directories, if any
if metadata.get("kind") == "package" and 'components' in metadata:
components = metadata.get("components")
for component in components:
message = ('Inspecting component %s.........\n' % component)
u.log_message(message,
log_file=session_file,
console=args.verbosity)
args.package_dir = os.path.join(package_dir, component)
create_package(args, api, command_obj, resume=resume)
args.package_dir = package_dir
else:
# create libraries or scripts
imports = []
category = str(metadata.get("category", DFT_CATEGORY))
if metadata.get("imports") is not None:
lib_imports = metadata.get("imports")
for lib_import in lib_imports:
args.package_dir = os.path.join(package_dir, lib_import)
if args.embed_libs:
library_ref = create_package( \
args, api, command_obj, resume=resume)
u.log_created_resources("imports", output_dir, library_ref)
else:
try:
# try to read the library id, if it is already there
library_ref = read_library_id(os.path.join( \
output_dir, os.path.basename(args.package_dir)))
except IOError:
library_ref = create_package( \
args, api, command_obj, resume=resume)
library_ref = read_library_id(os.path.join( \
output_dir, os.path.basename(args.package_dir)))
imports.append(library_ref)
args.package_dir = package_dir
# read the metadata.json information
message = ('Creating the %s.........\n' % metadata.get("kind"))
u.log_message(message, log_file=session_file, console=args.verbosity)
if metadata.get("kind") in WHIZZML_RESOURCES:
whizzml_code = os.path.normpath(os.path.join(args.package_dir, \
metadata.get("source_code", "%s.whizzml" % \
metadata.get("kind"))))
if args.embed_libs and metadata.get("kind") == WHIZZML_LIBRARY:
return whizzml_code
args.output_dir = os.path.join(output_dir, \
os.path.basename(package_dir))
# creating command to create the resource
command = COMMANDS[metadata.get("kind")] % (whizzml_code,
args.output_dir)
command_args = command.split()
bigml.util.check_dir(args.output_dir)
# getting inputs and outputs for the script from metadata
if "inputs" in metadata:
inputs_file = os.path.join(args.output_dir, "inputs.json")
u.write_to_utf8(inputs_file,
json.dumps(metadata.get("inputs")))
command_args.extend(["--declare-inputs", inputs_file])
if "outputs" in metadata:
outputs_file = os.path.join(args.output_dir, "outputs.json")
u.write_to_utf8(outputs_file,
json.dumps(metadata.get("outputs")))
command_args.extend(["--declare-outputs", outputs_file])
if "description" in metadata:
desc_file = os.path.join(args.output_dir, "description.txt")
u.write_to_utf8(desc_file, metadata.get("description"))
command_args.extend(["--description", desc_file])
if metadata.get("name"):
command_args.extend(["--name", metadata.get("name")])
if args.tag:
for tag in args.tag:
command_args.extend(["--tag", tag])
command_args.extend(["--category", category])
# adding imports, if any
if imports:
if args.embed_libs:
# imports to be embedded are in the same output directory
command_args.extend( \
["--embedded-imports", os.path.join(output_dir,
"imports")])
else:
# imports to be refereced by ID
command_args.extend(["--imports", ",".join(imports)])
command_args.extend(["--verbosity", str(args.verbosity)])
command_obj.propagate(command_args)
# u.add_api_context(command_args, args)
if args.upgrade:
command_args.extend(["--upgrade"])
if resume:
next_command = subcommand_list.pop()
if different_command(next_command, command):
resume = False
u.sys_log_message(command, log_file=subcommand_file)
execute_dispatcher(args=command_args)
elif not subcommand_list:
execute_dispatcher(args=['execute', '--resume'])
resume = False
else:
u.sys_log_message(command, log_file=subcommand_file)
execute_dispatcher(args=command_args)
args.output_dir = output_dir
return whizzml_code
return ""
def create_package(args, api, common_options, resume=False):
"""Creates the package whizzml resources as referred in the metadata.json
file.
"""
set_subcommand_file(args.output_dir)
if resume:
retrieve_subcommands()
# read the metadata.json information
message = ('Reading the metadata.json files.........\n')
u.log_message(message, log_file=session_file,
console=args.verbosity)
package_dir = args.package_dir
output_dir = args.output_dir
metadata_file = os.path.join(package_dir, METADATA_FILE)
metadata = None
with open(metadata_file) as metadata_handler:
metadata = json.load(metadata_handler)
# recurse into components/directories, if any
if metadata.get("kind") == "package" and 'components' in metadata:
components = metadata.get("components")
for component in components:
message = ('Inspecting component %s.........\n' % component)
u.log_message(message, log_file=session_file,
console=args.verbosity)
args.package_dir = os.path.join(package_dir, component)
create_package(args, api, common_options, resume=resume)
args.package_dir = package_dir
else:
# create libraries or scripts
imports = []
if metadata.get("imports") is not None:
lib_imports = metadata.get("imports")
for lib_import in lib_imports:
args.package_dir = os.path.join(package_dir, lib_import)
# try to read the library id, if it is already there
try:
library_id = read_library_id(os.path.join( \
output_dir, os.path.basename(args.package_dir)))
except IOError:
create_package(args, api, common_options, resume=resume)
library_id = read_library_id(os.path.join( \
output_dir, os.path.basename(args.package_dir)))
imports.append(library_id)
args.package_dir = package_dir
# read the metadata.json information
message = ('Creating the %s.........\n' % metadata.get("kind"))
u.log_message(message, log_file=session_file,
console=args.verbosity)
if metadata.get("kind") in WHIZZML_RESOURCES:
whizzml_code = os.path.join(args.package_dir, \
metadata.get("source_code", "%s.whizzml" % \
metadata.get("kind")))
args.output_dir = os.path.join(output_dir, \
os.path.basename(package_dir))
# creating command to create the resource
command = COMMANDS[metadata.get("kind")] % (whizzml_code,
args.output_dir)
command_args = command.split()
bigml.util.check_dir(args.output_dir)
# getting inputs and outputs for the script from metadata
if "inputs" in metadata:
inputs_file = os.path.join(args.output_dir, "inputs.json")
with open(inputs_file, "w") as inputs_handler:
json.dump(metadata.get("inputs"), inputs_handler)
command_args.extend(["--declare-inputs", inputs_file])
if "outputs" in metadata:
outputs_file = os.path.join(args.output_dir, "outputs.json")
with open(outputs_file, "w") as outputs_handler:
json.dump(metadata.get("outputs"), outputs_handler)
command_args.extend(["--declare-outputs", outputs_file])
if "description" in metadata:
desc_file = os.path.join(args.output_dir, "description.txt")
with open(desc_file, "w") as desc_handler:
desc_handler.write(metadata.get("description"))
command_args.extend(["--description", desc_file])
if metadata.get("name"):
command_args.extend(["--name", metadata.get("name")])
# adding imports, if any
if imports:
command_args.extend(["--imports", ",".join(imports)])
command_args.extend(["--verbosity", str(args.verbosity)])
if resume:
next_command = subcommand_list.pop()
if different_command(next_command, command):
resume = False
u.sys_log_message(command, log_file=subcommand_file)
execute_dispatcher(args=command_args)
elif not subcommand_list:
execute_dispatcher(args=['execute', '--resume'])
resume = False
else:
u.sys_log_message(command, log_file=subcommand_file)
execute_dispatcher(args=command_args)
args.output_dir = output_dir
def main(args=sys.argv[1:]):
"""Main process
"""
if args:
if not args[0].lower() in SUBCOMMANDS:
new_args = ["main"]
new_args.extend(args)
else:
new_args = args
# checks if the old --delete syntax is used
new_args = check_delete_option(new_args)
if not PYTHON3:
new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args]
if new_args[0] == "main":
main_dispatcher(args=new_args)
elif new_args[0] == "analyze":
analyze_dispatcher(args=new_args)
elif new_args[0] == "cluster":
cluster_dispatcher(args=new_args)
elif new_args[0] == "anomaly":
anomaly_dispatcher(args=new_args)
elif new_args[0] == "sample":
sample_dispatcher(args=new_args)
elif new_args[0] == "report":
report_dispatcher(args=new_args)
elif new_args[0] == "reify":
reify_dispatcher(args=new_args)
elif new_args[0] == "execute":
execute_dispatcher(args=new_args)
elif new_args[0] == "delete":
delete_dispatcher(args=new_args)
elif new_args[0] == "project":
project_dispatcher(args=new_args)
elif new_args[0] == "association":
association_dispatcher(args=new_args)
elif new_args[0] == "logistic-regression":
logistic_regression_dispatcher(args=new_args)
elif new_args[0] == "topic-model":
if no_stemmer:
sys.exit("To use the bigmler topic-model command you need the"
" Pystemmer library. Please, install it and"
" retry your command.")
topic_model_dispatcher(args=new_args)
elif new_args[0] == "time-series":
time_series_dispatcher(args=new_args)
elif new_args[0] == "deepnet":
deepnet_dispatcher(args=new_args)
elif new_args[0] == "whizzml":
whizzml_dispatcher(args=new_args)
elif new_args[0] == "export":
export_dispatcher(args=new_args)
elif new_args[0] == "retrain":
retrain_dispatcher(args=new_args)
else:
sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor"
"\n\nbigmler sample --help\n\n"
"\n\nbigmler analyze --help\n\n"
"\n\nbigmler cluster --help\n\n"
"\n\nbigmler anomaly --help\n\n"
"\n\nbigmler report --help\n\n"
"\n\nbigmler reify --help\n\n"
"\n\nbigmler project --help\n\n"
"\n\nbigmler association --help\n\n"
"\n\nbigmler logistic-regression --help\n\n"
"\n\nbigmler topic-model --help\n\n"
"\n\nbigmler time-series --help\n\n"
"\n\nbigmler deepnet --help\n\n"
"\n\nbigmler execute --help\n\n"
"\n\nbigmler whizzml --help\n\n"
"\n\nbigmler export --help\n\n"
"\n\nbigmler retrain --help\n\n"
"\n\nbigmler delete --help\n\n"
" for a list of options")
def main(args=sys.argv[1:]):
"""Main process
"""
if args:
if not args[0].lower() in SUBCOMMANDS:
new_args = ["main"]
new_args.extend(args)
else:
new_args = args
# checks if the old --delete syntax is used
new_args = check_delete_option(new_args)
if not PYTHON3:
new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args]
if new_args[0] == "main":
main_dispatcher(args=new_args)
elif new_args[0] == "analyze":
analyze_dispatcher(args=new_args)
elif new_args[0] == "cluster":
cluster_dispatcher(args=new_args)
elif new_args[0] == "anomaly":
anomaly_dispatcher(args=new_args)
elif new_args[0] == "sample":
sample_dispatcher(args=new_args)
elif new_args[0] == "report":
report_dispatcher(args=new_args)
elif new_args[0] == "reify":
reify_dispatcher(args=new_args)
elif new_args[0] == "execute":
execute_dispatcher(args=new_args)
elif new_args[0] == "delete":
delete_dispatcher(args=new_args)
elif new_args[0] == "project":
project_dispatcher(args=new_args)
elif new_args[0] == "association":
association_dispatcher(args=new_args)
elif new_args[0] == "logistic-regression":
logistic_regression_dispatcher(args=new_args)
elif new_args[0] == "topic-model":
if no_stemmer:
sys.exit("To use the bigmler topic-model command you need the"
" Pystemmer library. Please, install it and"
" retry your command.")
topic_model_dispatcher(args=new_args)
elif new_args[0] == "time-series":
time_series_dispatcher(args=new_args)
elif new_args[0] == "deepnet":
deepnet_dispatcher(args=new_args)
elif new_args[0] == "whizzml":
whizzml_dispatcher(args=new_args)
elif new_args[0] == "export":
export_dispatcher(args=new_args)
elif new_args[0] == "retrain":
retrain_dispatcher(args=new_args)
else:
sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor"
"\n\nbigmler sample --help\n\n"
"\n\nbigmler analyze --help\n\n"
"\n\nbigmler cluster --help\n\n"
"\n\nbigmler anomaly --help\n\n"
"\n\nbigmler report --help\n\n"
"\n\nbigmler reify --help\n\n"
"\n\nbigmler project --help\n\n"
"\n\nbigmler association --help\n\n"
"\n\nbigmler logistic-regression --help\n\n"
"\n\nbigmler topic-model --help\n\n"
"\n\nbigmler time-series --help\n\n"
"\n\nbigmler deepnet --help\n\n"
"\n\nbigmler execute --help\n\n"
"\n\nbigmler whizzml --help\n\n"
"\n\nbigmler export --help\n\n"
"\n\nbigmler retrain --help\n\n"
"\n\nbigmler delete --help\n\n"
" for a list of options")
