def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) else: sys.exit( "BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options" )
def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) else: sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options")
def create_package(args, api, common_options, resume=False): """Creates the package whizzml resources as referred in the metadata.json file. """ set_subcommand_file(args.output_dir) if resume: retrieve_subcommands() # read the metadata.json information message = ('Reading the metadata.json files.........\n') u.log_message(message, log_file=session_file, console=args.verbosity) package_dir = args.package_dir output_dir = args.output_dir metadata_file = os.path.join(package_dir, METADATA_FILE) metadata = None created_resources = [] with open(metadata_file) as metadata_handler: metadata = json.load(metadata_handler) # recurse into components/directories, if any if metadata.get("kind") == "package" and 'components' in metadata: components = metadata.get("components") for component in components: message = ('Inspecting component %s.........\n' % component) u.log_message(message, log_file=session_file, console=args.verbosity) args.package_dir = os.path.join(package_dir, component) create_package(args, api, common_options, resume=resume) args.package_dir = package_dir else: # create libraries or scripts imports = [] if metadata.get("imports") is not None: lib_imports = metadata.get("imports") for lib_import in lib_imports: args.package_dir = os.path.join(package_dir, lib_import) create_package(args, api, common_options, resume=resume) imports.append(read_library_id(os.path.join( \ output_dir, os.path.basename(args.package_dir)))) args.package_dir = package_dir # read the metadata.json information message = ('Creating the %s.........\n' % metadata.get("kind")) u.log_message(message, log_file=session_file, console=args.verbosity) if metadata.get("kind") in WHIZZML_RESOURCES: whizzml_code = os.path.join(args.package_dir, \ metadata.get("source_code", "%s.whizzml" % \ metadata.get("kind"))) args.output_dir = os.path.join(output_dir, \ os.path.basename(package_dir)) # creating command to create the resource command = COMMANDS[metadata.get("kind")] % (whizzml_code, args.output_dir) command_args = command.split() bigml.util.check_dir(args.output_dir) # getting inputs and outputs for the script from metadata if "inputs" in metadata: inputs_file = os.path.join(args.output_dir, "inputs.json") with open(inputs_file, "w") as inputs_handler: json.dump(metadata.get("inputs"), inputs_handler) command_args.extend(["--declare-inputs", inputs_file]) if "outputs" in metadata: outputs_file = os.path.join(args.output_dir, "outputs.json") with open(outputs_file, "w") as outputs_handler: json.dump(metadata.get("outputs"), outputs_handler) command_args.extend(["--declare-outputs", outputs_file]) if "description" in metadata: desc_file = os.path.join(args.output_dir, "description.txt") with open(desc_file, "w") as desc_handler: desc_handler.write(metadata.get("description")) command_args.extend(["--description", desc_file]) if metadata.get("name"): command_args.extend(["--name", metadata.get("name")]) # adding imports, if any if imports: command_args.extend(["--imports", ",".join(imports)]) command_args.extend(["--verbosity", str(args.verbosity)]) if resume: next_command = subcommand_list.pop() if different_command(next_command, command): resume = False u.sys_log_message(command, log_file=subcommand_file) execute_dispatcher(args=command_args) elif not subcommand_list: execute_dispatcher(args=['execute', '--resume']) resume = False else: u.sys_log_message(command, log_file=subcommand_file) execute_dispatcher(args=command_args) args.output_dir = output_dir
def create_package(args, api, command_obj, resume=False): """Creates the package whizzml resources as referred in the metadata.json file. """ set_subcommand_file(args.output_dir) if resume: retrieve_subcommands() # read the metadata.json information message = ('Reading the metadata.json files.........\n') u.log_message(message, log_file=session_file, console=args.verbosity) package_dir = args.package_dir output_dir = args.output_dir metadata_file = os.path.join(package_dir, METADATA_FILE) metadata = None with open(metadata_file) as metadata_handler: metadata = json.load(metadata_handler) # recurse into components/directories, if any if metadata.get("kind") == "package" and 'components' in metadata: components = metadata.get("components") for component in components: message = ('Inspecting component %s.........\n' % component) u.log_message(message, log_file=session_file, console=args.verbosity) args.package_dir = os.path.join(package_dir, component) create_package(args, api, command_obj, resume=resume) args.package_dir = package_dir else: # create libraries or scripts imports = [] category = str(metadata.get("category", DFT_CATEGORY)) if metadata.get("imports") is not None: lib_imports = metadata.get("imports") for lib_import in lib_imports: args.package_dir = os.path.join(package_dir, lib_import) if args.embed_libs: library_ref = create_package( \ args, api, command_obj, resume=resume) u.log_created_resources("imports", output_dir, library_ref) else: try: # try to read the library id, if it is already there library_ref = read_library_id(os.path.join( \ output_dir, os.path.basename(args.package_dir))) except IOError: library_ref = create_package( \ args, api, command_obj, resume=resume) library_ref = read_library_id(os.path.join( \ output_dir, os.path.basename(args.package_dir))) imports.append(library_ref) args.package_dir = package_dir # read the metadata.json information message = ('Creating the %s.........\n' % metadata.get("kind")) u.log_message(message, log_file=session_file, console=args.verbosity) if metadata.get("kind") in WHIZZML_RESOURCES: whizzml_code = os.path.normpath(os.path.join(args.package_dir, \ metadata.get("source_code", "%s.whizzml" % \ metadata.get("kind")))) if args.embed_libs and metadata.get("kind") == WHIZZML_LIBRARY: return whizzml_code args.output_dir = os.path.join(output_dir, \ os.path.basename(package_dir)) # creating command to create the resource command = COMMANDS[metadata.get("kind")] % (whizzml_code, args.output_dir) command_args = command.split() bigml.util.check_dir(args.output_dir) # getting inputs and outputs for the script from metadata if "inputs" in metadata: inputs_file = os.path.join(args.output_dir, "inputs.json") u.write_to_utf8(inputs_file, json.dumps(metadata.get("inputs"))) command_args.extend(["--declare-inputs", inputs_file]) if "outputs" in metadata: outputs_file = os.path.join(args.output_dir, "outputs.json") u.write_to_utf8(outputs_file, json.dumps(metadata.get("outputs"))) command_args.extend(["--declare-outputs", outputs_file]) if "description" in metadata: desc_file = os.path.join(args.output_dir, "description.txt") u.write_to_utf8(desc_file, metadata.get("description")) command_args.extend(["--description", desc_file]) if metadata.get("name"): command_args.extend(["--name", metadata.get("name")]) if args.tag: for tag in args.tag: command_args.extend(["--tag", tag]) command_args.extend(["--category", category]) # adding imports, if any if imports: if args.embed_libs: # imports to be embedded are in the same output directory command_args.extend( \ ["--embedded-imports", os.path.join(output_dir, "imports")]) else: # imports to be refereced by ID command_args.extend(["--imports", ",".join(imports)]) command_args.extend(["--verbosity", str(args.verbosity)]) command_obj.propagate(command_args) # u.add_api_context(command_args, args) if args.upgrade: command_args.extend(["--upgrade"]) if resume: next_command = subcommand_list.pop() if different_command(next_command, command): resume = False u.sys_log_message(command, log_file=subcommand_file) execute_dispatcher(args=command_args) elif not subcommand_list: execute_dispatcher(args=['execute', '--resume']) resume = False else: u.sys_log_message(command, log_file=subcommand_file) execute_dispatcher(args=command_args) args.output_dir = output_dir return whizzml_code return ""
def create_package(args, api, common_options, resume=False): """Creates the package whizzml resources as referred in the metadata.json file. """ set_subcommand_file(args.output_dir) if resume: retrieve_subcommands() # read the metadata.json information message = ('Reading the metadata.json files.........\n') u.log_message(message, log_file=session_file, console=args.verbosity) package_dir = args.package_dir output_dir = args.output_dir metadata_file = os.path.join(package_dir, METADATA_FILE) metadata = None with open(metadata_file) as metadata_handler: metadata = json.load(metadata_handler) # recurse into components/directories, if any if metadata.get("kind") == "package" and 'components' in metadata: components = metadata.get("components") for component in components: message = ('Inspecting component %s.........\n' % component) u.log_message(message, log_file=session_file, console=args.verbosity) args.package_dir = os.path.join(package_dir, component) create_package(args, api, common_options, resume=resume) args.package_dir = package_dir else: # create libraries or scripts imports = [] if metadata.get("imports") is not None: lib_imports = metadata.get("imports") for lib_import in lib_imports: args.package_dir = os.path.join(package_dir, lib_import) # try to read the library id, if it is already there try: library_id = read_library_id(os.path.join( \ output_dir, os.path.basename(args.package_dir))) except IOError: create_package(args, api, common_options, resume=resume) library_id = read_library_id(os.path.join( \ output_dir, os.path.basename(args.package_dir))) imports.append(library_id) args.package_dir = package_dir # read the metadata.json information message = ('Creating the %s.........\n' % metadata.get("kind")) u.log_message(message, log_file=session_file, console=args.verbosity) if metadata.get("kind") in WHIZZML_RESOURCES: whizzml_code = os.path.join(args.package_dir, \ metadata.get("source_code", "%s.whizzml" % \ metadata.get("kind"))) args.output_dir = os.path.join(output_dir, \ os.path.basename(package_dir)) # creating command to create the resource command = COMMANDS[metadata.get("kind")] % (whizzml_code, args.output_dir) command_args = command.split() bigml.util.check_dir(args.output_dir) # getting inputs and outputs for the script from metadata if "inputs" in metadata: inputs_file = os.path.join(args.output_dir, "inputs.json") with open(inputs_file, "w") as inputs_handler: json.dump(metadata.get("inputs"), inputs_handler) command_args.extend(["--declare-inputs", inputs_file]) if "outputs" in metadata: outputs_file = os.path.join(args.output_dir, "outputs.json") with open(outputs_file, "w") as outputs_handler: json.dump(metadata.get("outputs"), outputs_handler) command_args.extend(["--declare-outputs", outputs_file]) if "description" in metadata: desc_file = os.path.join(args.output_dir, "description.txt") with open(desc_file, "w") as desc_handler: desc_handler.write(metadata.get("description")) command_args.extend(["--description", desc_file]) if metadata.get("name"): command_args.extend(["--name", metadata.get("name")]) # adding imports, if any if imports: command_args.extend(["--imports", ",".join(imports)]) command_args.extend(["--verbosity", str(args.verbosity)]) if resume: next_command = subcommand_list.pop() if different_command(next_command, command): resume = False u.sys_log_message(command, log_file=subcommand_file) execute_dispatcher(args=command_args) elif not subcommand_list: execute_dispatcher(args=['execute', '--resume']) resume = False else: u.sys_log_message(command, log_file=subcommand_file) execute_dispatcher(args=command_args) args.output_dir = output_dir
def main(args=sys.argv[1:]): """Main process """ if args: if not args[0].lower() in SUBCOMMANDS: new_args = ["main"] new_args.extend(args) else: new_args = args # checks if the old --delete syntax is used new_args = check_delete_option(new_args) if not PYTHON3: new_args = [arg.decode(SYSTEM_ENCODING) for arg in new_args] if new_args[0] == "main": main_dispatcher(args=new_args) elif new_args[0] == "analyze": analyze_dispatcher(args=new_args) elif new_args[0] == "cluster": cluster_dispatcher(args=new_args) elif new_args[0] == "anomaly": anomaly_dispatcher(args=new_args) elif new_args[0] == "sample": sample_dispatcher(args=new_args) elif new_args[0] == "report": report_dispatcher(args=new_args) elif new_args[0] == "reify": reify_dispatcher(args=new_args) elif new_args[0] == "execute": execute_dispatcher(args=new_args) elif new_args[0] == "delete": delete_dispatcher(args=new_args) elif new_args[0] == "project": project_dispatcher(args=new_args) elif new_args[0] == "association": association_dispatcher(args=new_args) elif new_args[0] == "logistic-regression": logistic_regression_dispatcher(args=new_args) elif new_args[0] == "topic-model": if no_stemmer: sys.exit("To use the bigmler topic-model command you need the" " Pystemmer library. Please, install it and" " retry your command.") topic_model_dispatcher(args=new_args) elif new_args[0] == "time-series": time_series_dispatcher(args=new_args) elif new_args[0] == "deepnet": deepnet_dispatcher(args=new_args) elif new_args[0] == "whizzml": whizzml_dispatcher(args=new_args) elif new_args[0] == "export": export_dispatcher(args=new_args) elif new_args[0] == "retrain": retrain_dispatcher(args=new_args) else: sys.exit("BigMLer used with no arguments. Check:\nbigmler --help\n\nor" "\n\nbigmler sample --help\n\n" "\n\nbigmler analyze --help\n\n" "\n\nbigmler cluster --help\n\n" "\n\nbigmler anomaly --help\n\n" "\n\nbigmler report --help\n\n" "\n\nbigmler reify --help\n\n" "\n\nbigmler project --help\n\n" "\n\nbigmler association --help\n\n" "\n\nbigmler logistic-regression --help\n\n" "\n\nbigmler topic-model --help\n\n" "\n\nbigmler time-series --help\n\n" "\n\nbigmler deepnet --help\n\n" "\n\nbigmler execute --help\n\n" "\n\nbigmler whizzml --help\n\n" "\n\nbigmler export --help\n\n" "\n\nbigmler retrain --help\n\n" "\n\nbigmler delete --help\n\n" " for a list of options")