def execute_using_docker(mint_config_file: Path): model_path = mint_config_file.parent name = model_path.name docker_path = model_path / DOCKER_DIR try: image = build_docker(docker_path, name) except APIError as e: click.secho("Error building the image", fg="red") click.echo(e) exit(1) except Exception as e: click.secho("Error building the image", fg="red") click.echo(e) exit(1) now = datetime.now().timestamp() src_dir = create_execution_directory(mint_config_file, model_path) try: resource = create_model_catalog_resource(mint_config_file) except ValueError: logging.error(exc_info=True) pass docker_run(image, resource, src_dir) detect_news_file(src_dir, mint_config_file, now) write_spec(mint_config_file, LAST_EXECUTION_DIR, str(src_dir.absolute())) return src_dir
def step3(mic_file): """ Create MINT wrapper using the mic.yaml. This command will handle adding inputs and parameters into the run file. - You must pass the MIC_FILE (mic.yaml) using the option (-f) or run the command from the same directory as mic.yaml Example: mic encapsulate step3 -f <mic_file> """ if not Path(mic_file).exists(): click.secho("Error: {} doesn't exists".format(mic_file), fg="red") exit(1) mic_config_path = Path(mic_file) model_directory_path = mic_config_path.parent inputs, parameters, outputs, configs = get_inputs_parameters( mic_config_path) number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters( mic_config_path) run_path = render_run_sh(model_directory_path, inputs, parameters, number_inputs, number_parameters) render_io_sh(model_directory_path, inputs, parameters, configs) render_output(model_directory_path, [], False) write_spec(mic_config_path, STEP_KEY, 3) click.echo("The MIC Wrapper has been created at: {}".format(run_path)) click.secho( "Before the next step you must add any (bash) commands needed to run your model between the two " "comments in the wrapper file. This file is located in {}/{}".format( SRC_DIR, RUN_FILE), fg="green") click.secho( "If your model has a configuration file, you will need to edit the values to match {}\'s parameter " "names then run step4. Otherwise you can move on to step5. See the docs for more details" "".format(CONFIG_YAML_NAME), fg="green")
def publish_docker(mic_config_path, profile): # DOCKER_KEY doesn't have the docker username docker_image = get_key_spec(mic_config_path, DOCKER_KEY) try: docker_image = docker_image.split('/')[-1] except: pass try: docker_image = docker_image.split(':')[0] except: pass container_name = get_key_spec(mic_config_path, CONTAINER_NAME_KEY) docker_username = get_docker_username(profile) version = get_key_spec(mic_config_path, VERSION_KEY) docker_image_with_version = f"""{docker_username}/{docker_image}:{version}""" docker_container_cmd = f"""docker container commit {container_name} {docker_image_with_version} """ click.secho(f"Committing the changes into the Docker Image " f"Please wait...") os.system(docker_container_cmd) click.secho("Uploading the Docker Image") logging.info("Publish docker image") try: docker_push_cmd = f"""docker push {docker_image_with_version} """ os.system(docker_push_cmd) write_spec(mic_config_path, DOCKER_KEY, docker_image_with_version) except Exception as e: raise e
def outputs(mic_file, custom_outputs): """ Describe the outputs of your model using the information obtained by the `trace` command. To identify which inputs have been automatically detected, execute `mic pkg outputs -f mic/mic.yaml` and then inspect the mic.yaml file - You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option; or run the command from the same directory as mic.yaml - Identify undetected files or directories in the mic.yaml file and pass them as as arguments to the command mic pkg outputs -f <mic_file> [undetected files]... Example: mic pkg outputs -f mic/mic.yaml output.txt outputs_directory """ # Searches for mic file if user does not provide one mic_file = check_mic_path(mic_file) log_command(logging, "outputs", mic_file=mic_file, custom_outputs=custom_outputs) try: info_start_outputs() mic_config_file = Path(mic_file) user_execution_directory = mic_config_file.parent.parent repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR, user_execution_directory) repro_zip_trace_dir = Path(repro_zip_trace_dir) repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE spec = get_spec(repro_zip_config_file) custom_outputs = [ str(user_execution_directory / Path(i).relative_to(user_execution_directory)) for i in list(custom_outputs) ] outputs = get_outputs_reprozip(spec, user_execution_directory) logging.debug("Outputs found from reprozip: {}".format(outputs)) for i in list(custom_outputs): if Path(i).is_dir(): outputs += get_filepaths(i) else: outputs.append(i) for i in outputs: click.secho(f"""Output added: {Path(i).name} """, fg="blue") info_end_outputs(outputs) write_spec(mic_config_file, OUTPUTS_KEY, relative(outputs, user_execution_directory)) logging.info("outputs done") except Exception as e: logging.exception(f"Outputs failed: {e}") click.secho("Failed", fg="red") click.secho(e)
def step6(mic_file): """ Build and run the Docker image Example: mic encapsulate step6 -f <mic_file> """ mic_config_path = Path(mic_file) execute_using_docker(Path(mic_file)) write_spec(mic_config_path, STEP_KEY, 6) click.secho("Success", fg="green")
def step7(mic_file, profile): """ Publish your code and MIC wrapper on GitHub and the Docker Image on DockerHub Example: mic encapsulate step7 -f <mic_file> """ info_step8() mic_config_path = Path(mic_file) model_dir = mic_config_path.parent click.secho("Deleting the executions") push(model_dir, mic_config_path, profile) publish_docker(mic_config_path, profile) write_spec(mic_config_path, STEP_KEY, 7)
def push(model_directory: Path, mic_config_path: Path, name: str, profile): repository_name = name _version = new_version() write_spec(mic_config_path, VERSION_KEY, _version) logging.info("Compressing code") click.secho("Compressing your code") zip_file = compress_src_dir(model_directory, _version) url = upload_file(zip_file, profile, "components") write_spec(mic_config_path, MINT_COMPONENT_KEY, url) logging.info("Push complete: {}".format({ 'repository': url, 'version': _version })) click.secho("Repository: {}".format(url)) click.secho("Version: {}".format(_version))
def step5(mic_file): """ Set up Docker Image for model. Example: mic encapsulate step5 -f <mic_file> """ mic_config_path = Path(mic_file) model_dir = mic_config_path.parent src_dir_path = model_dir / SRC_DIR if not mic_config_path.exists(): exit(1) frameworks = detect_framework(src_dir_path) if len(frameworks) > 1: click.secho( "MIC has detect {} possible framework/language on component: {}". format(len(frameworks), ",".join([i.label for i in frameworks]))) click.secho("Please select the correct option") click.secho( "This information allows MIC to select the correct Docker Image") framework = click.prompt("Select a option ".format(Framework), show_choices=True, type=click.Choice(frameworks, case_sensitive=False), value_proc=handle) elif len(frameworks) == 1: framework = frameworks[0] else: framework = Framework.GENERIC if framework == Framework.GENERIC: bin_dir = model_dir / DOCKER_DIR / "bin" bin_dir.mkdir(exist_ok=True) elif framework == Framework.PYTHON37: requirements_file = Path( mic_file).parent / DOCKER_DIR / REQUIREMENTS_FILE freeze(requirements_file) click.echo( "Extracting the Python dependencies.\nYou can view or edit the dependencies file {} " .format(requirements_file)) dockerfile = render_dockerfile(model_dir, framework) click.secho("Dockerfile has been created: {}".format(dockerfile)) write_spec(mic_config_path, STEP_KEY, 5)
def step8(mic_file, profile): """ Publish your model into the Model Catalog Example: mic encapsulate step8 -f <mic_file> """ mic_config_path = Path(mic_file) model_configuration = create_model_catalog_resource(Path(mic_file), allow_local_path=False) api_response_model, api_response_mc = publish_model_configuration( model_configuration, profile) click.echo( "You can run or see the details using DAME. More info at https://dame-cli.readthedocs.io/en/latest/" ) click.echo("For example, you can run it using:\ndame run {}".format( obtain_id(api_response_mc.id))) write_spec(mic_config_path, STEP_KEY, 8)
def step4(mic_file, configuration_files): """ If your model does not use configuration files, you can skip this step Specify the inputs and parameters of your model component from configuration file(s) - You must pass the MIC_FILE (mic.yaml) using the option (-f) or run the command from the same directory as mic.yaml - Pass the configuration files as arguments mic encapsulate step4 -f <mic_file> [configuration_files]... Example: mic encapsulate step4 -f mic.yaml data/example_dir/file1.txt data/file2.txt """ mic_config_path = Path(mic_file) if not mic_config_path.exists(): click.secho("Error: that configuration path does not exist", fg="red") exit(1) # loop through configuration_files list for cp in configuration_files: # The program will crash if the users configuration file is not in the data dir. This checks for that if DATA_DIR not in Path(cp).absolute().parts: click.secho( "Error: Configuration file must be stored within {} directory". format(DATA_DIR), fg="red") click.secho("Bad path input: {}".format(cp)) exit(1) add_configuration_files(mic_config_path, configuration_files) model_directory_path = mic_config_path.parent inputs, parameters, outputs, configs = get_inputs_parameters( mic_config_path) number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters( mic_config_path) render_run_sh(model_directory_path, inputs, parameters, number_inputs, number_parameters) render_io_sh(model_directory_path, inputs, parameters, configs) render_output(model_directory_path, [], False) write_spec(mic_config_path, STEP_KEY, 4)
def publish_docker(mic_config_path, profile): version = get_key_spec(mic_config_path, VERSION_KEY) image_name = get_key_spec(mic_config_path, DOCKER_KEY) credentials = get_credentials(profile) click.secho("Publishing the Docker Image") try: client = docker.from_env() if DOCKER_USERNAME_KEY not in credentials: exit(0) username = credentials[DOCKER_USERNAME_KEY] image = client.images.get(image_name) image_name_without_version = image_name.split("/")[-1].split(':')[0] repository = "{}/{}".format(username, image_name_without_version) image.tag(repository, version) client.images.push(repository, version) except Exception as e: raise e docker_image = "{}:{}".format(repository, version) click.secho("Docker Image: {}".format(docker_image)) write_spec(mic_config_path, DOCKER_KEY, docker_image)
def push(model_directory: Path, mic_config_path: Path, profile): click.secho("Creating the git repository") repo = get_or_create_repo(model_directory) click.secho("Compressing your code") compress_src_dir(model_directory) click.secho("Creating a new commit") git_commit(repo) click.secho("Creating or using the GitHub repository") url = check_create_remote_repo(repo, profile, model_directory.name, model_directory) click.secho("Creating a new version") _version = git_tag(repo, author) click.secho("Pushing your changes to the server") git_push(repo, profile, _version) repo = get_github_repo(profile, model_directory.name) for i in repo.get_contents(""): if i.name == "{}.zip".format(MINT_COMPONENT_ZIP): file = i break write_spec(mic_config_path, REPO_KEY, url) write_spec(mic_config_path, VERSION_KEY, _version) write_spec(mic_config_path, MINT_COMPONENT_KEY, file.download_url) click.secho("Repository: {}".format(url)) click.secho("Version: {}".format(_version)) remove_temp_files(mic_config_path)
def step2(mic_file, parameters): """ Fill the MIC configuration file with the information about the parameters and inputs MIC is going to detect: - the inputs (files and directory) and add them in the MIC configuration file. - the parameters and add them in the configuration file Example: mic encapsulate step2 -f <mic_file> -p <number_of_parameters> """ mic_config_path = Path(mic_file) inputs_dir = mic_config_path.parent / DATA_DIRECTORY_NAME if not inputs_dir.exists(): exit(1) fill_config_file_yaml(Path(mic_file), inputs_dir, parameters) click.secho( "MIC has added the parameters and inputs into the {} ({})".format( MIC_CONFIG_FILE_NAME, CONFIG_YAML_NAME)) click.secho("You can see the changes in {}".format( Path(mic_file).absolute()), fg="green") click.secho("Before step3: ", fg="green") click.secho( "You must add a default value for the \"default-value\" field in {}. Just replace the 0 with your value" "".format(CONFIG_YAML_NAME), fg="green") click.secho( "It is recommended you also add a description for each input and parameter in {}" .format(CONFIG_YAML_NAME), fg="green") click.secho( "If you use a script to initialize or create visualizations of your model you must copy these into the " "src directory", fg="green") write_spec(mic_config_path, STEP_KEY, 2)
def upload_component(cwl_document, cwl_values, profile): # create a temporal file cwl_document_path = Path(cwl_document) name = cwl_document_path.stem cwl_dir = cwl_document_path.parent mic_config_path = create_config_file_yaml(cwl_dir, f"""{name}_mic.yaml""") cwl_values_dict = get_spec(Path(cwl_values)) # get the name from cwl document and write write_spec(mic_config_path, NAME_KEY, name) # read the CWL document and stored as dict cwl_document_dict = get_spec(cwl_document_path) # get the input, parameters and outputs from CWL document inputs = get_inputs(cwl_document_dict) outputs = get_outputs(cwl_document_dict) parameters = get_parameters(cwl_document_dict) # write then on MIC file add_inputs(mic_config_path, inputs, cwl_values_dict) add_outputs(mic_config_path, outputs, cwl_values) add_parameters(mic_config_path, parameters, cwl_values_dict) # obtain the docker image from cwl docker_image = get_docker_image(cwl_document_dict) write_spec(mic_config_path, DOCKER_KEY, docker_image) #obtain cwl command write_spec(mic_config_path, CWL_KEY, cwl_document_path) # Message publish start #info_start_publish(True) # push the component model_configuration = create_model_catalog_resource_cwl( Path(mic_config_path), name, allow_local_path=False ) api_response_model, api_response_mc, model_id, software_version_id = publish_model_configuration( model_configuration, profile) # Message publish end info_end_publish(obtain_id(model_id), obtain_id(software_version_id), obtain_id(api_response_mc.id), profile )
def start(user_execution_directory, name, image): """ This step generates a mic.yaml file and the directories (data/, src/, docker/). The argument: `model_configuration_name` is the name of the model component you are defining in MIC """ user_execution_directory = Path(user_execution_directory) mic_dir = user_execution_directory / MIC_DIR create_base_directories(mic_dir) # Cant log the start command until the mic dir has been created. Else mic will think the directory already exists if make_log_file(): log_system_info(get_mic_logger().name) log_command(logging, "start", name=name, image=image) mic_config_path = create_config_file_yaml(mic_dir) custom_image = False if image is None: framework = detect_framework_main(user_execution_directory) else: # If a user provides a image, the framework is generic. custom_image = True framework = Framework.GENERIC framework.image = image os.system(f"docker pull {framework.image}") render_dockerfile(mic_dir, framework) # Make sure the name given is valid if not name.islower(): logging.debug( "User's model name does not contain all lowercase characters. Setting it to lower" ) click.secho( "Model name must be lower case. Mic will replace any uppercase letters", fg='yellow') name = name.lower() os.system(f"docker pull {framework.image}") try: user_image = build_docker(mic_dir / DOCKER_DIR, name) except ValueError: click.secho("The extraction of dependencies has failed", fg='red') user_image = framework.image container_name = f"{name}_{str(uuid.uuid4())[:8]}" write_spec(mic_config_path, NAME_KEY, name) write_spec(mic_config_path, DOCKER_KEY, user_image) write_spec(mic_config_path, FRAMEWORK_KEY, framework.label) write_spec(mic_config_path, CONTAINER_NAME_KEY, container_name) docker_cmd = f"""docker run -ti \ --name={container_name} \ --cap-add=SYS_PTRACE \ -v \"{user_execution_directory}\":/tmp/mint \ -w /tmp/mint {user_image} """ if custom_image: click.secho(f""" You are using a custom image Installing MIC and some dependencies """, fg="green") try: os.system(docker_cmd) logging.info("start done") except Exception as e: logging.exception(f"Start failed: {e}") click.secho("Failed", fg="red") click.secho(e)
def inputs(mic_file, custom_inputs): """ Describe the inputs of your model using the information obtained by the `trace` command. To identify which inputs have been automatically detected, execute `mic pkg inputs -f mic/mic.yaml` and then inspect the mic.yaml file - You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option or run the command from the same directory as mic.yaml - Identify undetected files in or directories in mic.yaml and add them as arguments to the `inputs` command mic pkg inputs -f <mic_file> [undetected files]... Usage example: mic pkg inputs -f mic/mic.yaml input.txt inputs_directory """ # Searches for mic file if user does not provide one mic_file = check_mic_path(mic_file) log_command(logging, "inputs", mic_file=mic_file, custom_inputs=custom_inputs) try: info_start_inputs() mic_config_file = Path(mic_file) mic_directory_path = mic_config_file.parent user_execution_directory = mic_config_file.parent.parent repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR, user_execution_directory) repro_zip_trace_dir = Path(repro_zip_trace_dir) repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE spec = get_spec(repro_zip_config_file) custom_inputs = [ str(user_execution_directory / Path(i).relative_to(user_execution_directory)) for i in list(custom_inputs) ] inputs_reprozip = get_inputs_outputs_reprozip( spec, user_execution_directory) logging.debug("Inputs found from reprozip: {}".format(inputs_reprozip)) # obtain config: if a file is a config cannot be a input config_files = get_configs(mic_config_file) config_files_list = [ str(user_execution_directory / item[PATH_KEY]) for key, item in config_files.items() ] if config_files else [] code_files = find_code_files(spec, inputs_reprozip, config_files_list, user_execution_directory) logging.debug("code files found from reprozip: {}".format(code_files)) new_inputs = [] inputs_reprozip += list(custom_inputs) data_dir = mic_directory_path.absolute() / DATA_DIR if data_dir.exists(): shutil.rmtree(data_dir) data_dir.mkdir() _outputs = get_outputs_reprozip(spec, user_execution_directory) for _input in inputs_reprozip: item = user_execution_directory / _input name = Path(_input).name if str(item) in config_files_list or str( item) in code_files or str(item) in _outputs: logging.info(f"Ignoring the config as an input: {item}") else: # Deleting the outputs of the inputs. if item.is_dir(): if sorted([str(i) for i in item.iterdir()]) == sorted(_outputs): logging.info(f"Skipping: {item}") else: logging.info( f"""Compressing the input directory ({name})""") zip_file = compress_directory( item, user_execution_directory) dst_dir = data_dir dst_file = dst_dir / Path(zip_file).name if dst_file.exists(): os.remove(dst_file) shutil.move(str(zip_file), str(dst_dir)) new_inputs.append(zip_file) click.secho(f"""Input {name} added """, fg="blue") logging.info("Input added: {}".format(name)) else: new_inputs.append(item) dst_file = mic_directory_path / DATA_DIR / str(item.name) shutil.copy(item, dst_file) click.secho(f"""Input {name} added """, fg="blue") logging.info("Input added: {}".format(name)) info_end_inputs(new_inputs) write_spec(mic_config_file, INPUTS_KEY, relative(new_inputs, user_execution_directory)) write_spec(mic_config_file, CODE_KEY, relative(code_files, user_execution_directory)) logging.info("inputs done") except Exception as e: logging.exception(f"Inputs failed: {e}") click.secho("Failed", fg="red") click.secho(e)
def configs(mic_file, configuration_files, auto_param): """ Note: If your model does not use configuration files, you can skip this step Specify which parameters of your model component you want to expose from any configuration file. - You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option or run the command from the same directory as mic.yaml - Pass your model configuration files as arguments mic pkg configs -f <mic_file> [configuration_files]... If you have manually changed some parameters, the -a option will attempt to recognize the configuration files automatically Example: mic pkg configs -f mic.yaml data/example_dir/file1.txt data/file2.txt """ # Searches for mic file if user does not provide one mic_file = check_mic_path(mic_file) log_command(logging, "configs", mic_file=mic_file, configuration_files=configuration_files, auto_param=auto_param) try: mic_config_file = Path(mic_file) user_execution_directory = mic_config_file.parent.parent if not mic_config_file.exists(): click.secho("Error: that configuration path does not exist", fg="red") logging.error("mic config file does not exist") exit(1) configuration_files = [ str(Path(x).absolute()) for x in list(configuration_files) ] try: # Add config file names to yaml write_spec(mic_config_file, CONFIG_FILE_KEY, relative(configuration_files, user_execution_directory)) except Exception as e: click.secho("Failed. Error message: {}".format(e), fg="red") logging.exception("Failed writing configs: {}".format(e)) for item in configuration_files: click.secho("Added: {} as a configuration file".format(item)) logging.info("Added config file: {}".format(item)) if auto_param: # Parse parameters from config file(s) and add them to mic.yaml add_params_from_config(mic_config_file, item) write_spec(mic_config_file, STEP_KEY, 2) logging.info("configs done") except Exception as e: logging.exception(f"Configs failed: {e}") click.secho("Failed", fg="red") click.secho(e)
def add_parameters(mic_file, name, value, overwrite, description): """ Add a parameter into the MIC file (mic.yaml). - You must pass the MIC file (mic.yaml) as an argument using the (-f) option; or run the command from the same directory as the MIC file (mic.yaml) Usage example: mic pkg parameters -f <mic_file> --name PARAMETER_NAME --value PARAMETER_VALUE """ # Searches for mic file if user does not provide one mic_file = check_mic_path(mic_file) log_command(logging, "add_parameters", mic_file=mic_file, name=name, value=value, overwrite=overwrite, description=description) try: path = Path(mic_file) spec = get_spec(path) if (name or value or description or overwrite) and not ( (name is not None) and (value is not None)): click.secho( "Must give name and value to manually add new parameter. Aborting", fg="yellow") logging.info("Invalid manual parameter given") exit(0) if PARAMETERS_KEY not in spec: spec[PARAMETERS_KEY] = {} # Automacically add parameters from trace command. Use heuristic "if item isnt file its a parameter" if name is None: click.echo("Automatically adding any parameters from trace") logging.info("Automatically adding any parameters from trace") mic_config_file = Path(mic_file) user_execution_directory = mic_config_file.parent.parent repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR, user_execution_directory) repro_zip_trace_dir = Path(repro_zip_trace_dir) repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE reprozip_spec = get_spec(repro_zip_config_file) spec = get_parameters_reprozip(spec, reprozip_spec) else: if not overwrite and name in spec[PARAMETERS_KEY]: click.echo( "The parameter exists. Add the option --overwrite to overwrite it." ) logging.info( "Parameter already exists. aborting because overwrite flag is false" ) exit(1) else: if description is None: description = "" type_value____name__ = type(value).__name__ click.echo( f"Adding the parameter {name}, value {value} and type {type_value____name__}" ) new_par = { name: { NAME_KEY: name, DEFAULT_VALUE_KEY: value, DATATYPE_KEY: type_value____name__, DEFAULT_DESCRIPTION_KEY: description } } logging.debug("Adding parameter: {}".format(new_par)) spec[PARAMETERS_KEY].update(new_par) write_spec(path, PARAMETERS_KEY, spec[PARAMETERS_KEY]) logging.info("add_parameters done") except Exception as e: logging.exception(f"Add Parameters failed: {e}") click.secho("Failed", fg="red") click.secho(e)