def wrapper(mic_file):
"""
Generates the MIC Wrapper:a directory structure and commands required to run your model component using the
information gathered from previous steps
- You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option or run the
command from the same directory as mic.yaml
mic pkg wrapper -f <mic_file>
Example:
mic pkg wrapper -f mic/mic.yaml
"""
# Searches for mic file if user does not provide one
mic_file = check_mic_path(mic_file)
log_command(logging, "wrapper", mic_file=mic_file)
try:
info_start_wrapper()
mic_config_file = Path(mic_file)
user_execution_directory = mic_config_file.parent.parent
repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR,
user_execution_directory)
repro_zip_trace_dir = Path(repro_zip_trace_dir)
repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE
mic_directory_path = mic_config_file.parent
mic_inputs = get_inputs(mic_config_file)
mic_parameters = get_parameters(mic_config_file)
mic_outputs = get_outputs_mic(mic_config_file)
mic_configs = get_configs(mic_config_file)
mic_code = get_code(mic_config_file)
spec = get_spec(mic_config_file)
reprozip_spec = get_spec(repro_zip_config_file)
logging.info("Generating wrapper code")
code = f"""{generate_pre_runner(spec, user_execution_directory)}
{generate_runner(reprozip_spec, user_execution_directory, mic_inputs, mic_outputs, mic_parameters)}"""
render_bash_color(mic_directory_path)
render_run_sh(mic_directory_path, mic_inputs, mic_parameters,
mic_outputs, code)
render_io_sh(mic_directory_path, mic_inputs, mic_parameters,
mic_configs)
render_output(mic_directory_path, mic_outputs, False)
copy_code_to_src(mic_code, user_execution_directory,
mic_directory_path / SRC_DIR)
copy_config_to_src(mic_configs, user_execution_directory,
mic_directory_path / SRC_DIR)
info_end_wrapper(mic_directory_path / SRC_DIR / RUN_FILE)
logging.info("wrapper done")
except Exception as e:
logging.exception(f"Wrapper failed: {e}")
click.secho("Failed", fg="red")
click.secho(e)
def upload_image(cwl_document, profile):
cwl_path = Path(cwl_document)
if not cwl_path.exists():
exit(1)
#Get the cwl spec
cwl_spec = get_spec(cwl_path)
try:
docker_image = get_docker_image(cwl_spec)
docker_image_parsed = parse_docker_name(docker_image)
except ValueError as e:
click.secho(f"""Unable to find image, please verify the {cwl_document}""")
exit(1)
image_exists(docker_image)
click.echo(f"""Image {docker_image} detected""")
#Get the DockerUsername from crendentials
docker_username = get_docker_username(profile)
#Generate a unique version from the time
version = datetime.now().strftime("%Y%m%d-%H%M%S")
click.echo(f"""Docker username detected: {docker_username}""")
docker_image_name = docker_image_parsed["image_name"]
docker_image_with_version = f"""{docker_username}/{docker_image_name}:{version}"""
docker_push_cmd = f"""docker tag {docker_image} {docker_image_with_version} && docker push {docker_image_with_version}"""
try:
os.system(docker_push_cmd)
except Exception as e:
click.echo('Unable to push the image')
exit(1)
raise e
update_docker_image(cwl_path, docker_image_with_version)
def test_param_detection():
mic_yaml = Path("254/mic/mic.yaml")
mic_spec = get_spec(Path(__file__).parent / RESOURCES / mic_yaml)
repro_yaml = Path("254/.reprozip-trace/config.yml")
repro_spec = get_spec(Path(__file__).parent / RESOURCES / repro_yaml)
spec = get_parameters_reprozip(mic_spec, repro_spec)
assert spec == {
"step": 1,
"name": "test-autoparam",
"docker_image": "test-autoparam:latest",
"inputs": {
"a_txt": {
"path": "a.txt",
"format": "txt"
},
"in_txt": {
"path": "in.txt",
"format": "txt"
}
},
"code_files": {
"addtoarray_sh": {
"path": "addtoarray.sh",
"format": "sh"
}
},
"outputs": {
"out_csv": {
"path": "outputs/out.csv",
"format": "csv"
}
},
"parameters": {
"param_1": {
"name": "",
"default_value": "15",
"type": "int",
"description": ""
}
}
}
def outputs(mic_file, custom_outputs):
"""
Describe the outputs of your model using the information obtained by the `trace` command.
To identify which inputs have been automatically detected, execute `mic pkg outputs -f mic/mic.yaml`
and then inspect the mic.yaml file
- You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option; or run the
command from the same directory as mic.yaml
- Identify undetected files or directories in the mic.yaml file and pass them as as arguments to the command
mic pkg outputs -f <mic_file> [undetected files]...
Example:
mic pkg outputs -f mic/mic.yaml output.txt outputs_directory
"""
# Searches for mic file if user does not provide one
mic_file = check_mic_path(mic_file)
log_command(logging,
"outputs",
mic_file=mic_file,
custom_outputs=custom_outputs)
try:
info_start_outputs()
mic_config_file = Path(mic_file)
user_execution_directory = mic_config_file.parent.parent
repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR,
user_execution_directory)
repro_zip_trace_dir = Path(repro_zip_trace_dir)
repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE
spec = get_spec(repro_zip_config_file)
custom_outputs = [
str(user_execution_directory /
Path(i).relative_to(user_execution_directory))
for i in list(custom_outputs)
]
outputs = get_outputs_reprozip(spec, user_execution_directory)
logging.debug("Outputs found from reprozip: {}".format(outputs))
for i in list(custom_outputs):
if Path(i).is_dir():
outputs += get_filepaths(i)
else:
outputs.append(i)
for i in outputs:
click.secho(f"""Output added: {Path(i).name} """, fg="blue")
info_end_outputs(outputs)
write_spec(mic_config_file, OUTPUTS_KEY,
relative(outputs, user_execution_directory))
logging.info("outputs done")
except Exception as e:
logging.exception(f"Outputs failed: {e}")
click.secho("Failed", fg="red")
click.secho(e)
def upload_component(cwl_document, cwl_values, profile):
# create a temporal file
cwl_document_path = Path(cwl_document)
name = cwl_document_path.stem
cwl_dir = cwl_document_path.parent
mic_config_path = create_config_file_yaml(cwl_dir, f"""{name}_mic.yaml""")
cwl_values_dict = get_spec(Path(cwl_values))
# get the name from cwl document and write
write_spec(mic_config_path, NAME_KEY, name)
# read the CWL document and stored as dict
cwl_document_dict = get_spec(cwl_document_path)
# get the input, parameters and outputs from CWL document
inputs = get_inputs(cwl_document_dict)
outputs = get_outputs(cwl_document_dict)
parameters = get_parameters(cwl_document_dict)
# write then on MIC file
add_inputs(mic_config_path, inputs, cwl_values_dict)
add_outputs(mic_config_path, outputs, cwl_values)
add_parameters(mic_config_path, parameters, cwl_values_dict)
# obtain the docker image from cwl
docker_image = get_docker_image(cwl_document_dict)
write_spec(mic_config_path, DOCKER_KEY, docker_image)
#obtain cwl command
write_spec(mic_config_path, CWL_KEY, cwl_document_path)
# Message publish start
#info_start_publish(True)
# push the component
model_configuration = create_model_catalog_resource_cwl(
Path(mic_config_path),
name,
allow_local_path=False
)
api_response_model, api_response_mc, model_id, software_version_id = publish_model_configuration(
model_configuration, profile)
# Message publish end
info_end_publish(obtain_id(model_id),
obtain_id(software_version_id),
obtain_id(api_response_mc.id),
profile
)
def test_wrapper_code():
mic_yaml = Path("254/mic/mic3.yaml")
repro_yaml = Path("254/.reprozip-trace2/config.yml")
repro_spec = get_spec(Path(__file__).parent / RESOURCES / repro_yaml)
inp = get_inputs(Path(__file__).parent / RESOURCES / mic_yaml)
outp = get_outputs_mic(Path(__file__).parent / RESOURCES / mic_yaml)
params = get_parameters(Path(__file__).parent / RESOURCES / mic_yaml)
code = generate_runner(repro_spec, DEFAULT_PATH, inp, outp, params)
assert code == "\npushd .\n/bin/sh ./addtoarray.sh ${a_txt} ${in_txt} ${param_1} \"${param_2}\" ${param_3} " \
"${param_4} \"${param_5}\" \"${param_6}\"\npopd"
def inputs(mic_file, custom_inputs):
"""
Describe the inputs of your model using the information obtained by the `trace` command. To identify which inputs have
been automatically detected, execute `mic pkg inputs -f mic/mic.yaml` and then inspect the mic.yaml file
- You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option or run the
command from the same directory as mic.yaml
- Identify undetected files in or directories in mic.yaml and add them as arguments to the `inputs` command
mic pkg inputs -f <mic_file> [undetected files]...
Usage example:
mic pkg inputs -f mic/mic.yaml input.txt inputs_directory
"""
# Searches for mic file if user does not provide one
mic_file = check_mic_path(mic_file)
log_command(logging,
"inputs",
mic_file=mic_file,
custom_inputs=custom_inputs)
try:
info_start_inputs()
mic_config_file = Path(mic_file)
mic_directory_path = mic_config_file.parent
user_execution_directory = mic_config_file.parent.parent
repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR,
user_execution_directory)
repro_zip_trace_dir = Path(repro_zip_trace_dir)
repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE
spec = get_spec(repro_zip_config_file)
custom_inputs = [
str(user_execution_directory /
Path(i).relative_to(user_execution_directory))
for i in list(custom_inputs)
]
inputs_reprozip = get_inputs_outputs_reprozip(
spec, user_execution_directory)
logging.debug("Inputs found from reprozip: {}".format(inputs_reprozip))
# obtain config: if a file is a config cannot be a input
config_files = get_configs(mic_config_file)
config_files_list = [
str(user_execution_directory / item[PATH_KEY])
for key, item in config_files.items()
] if config_files else []
code_files = find_code_files(spec, inputs_reprozip, config_files_list,
user_execution_directory)
logging.debug("code files found from reprozip: {}".format(code_files))
new_inputs = []
inputs_reprozip += list(custom_inputs)
data_dir = mic_directory_path.absolute() / DATA_DIR
if data_dir.exists():
shutil.rmtree(data_dir)
data_dir.mkdir()
_outputs = get_outputs_reprozip(spec, user_execution_directory)
for _input in inputs_reprozip:
item = user_execution_directory / _input
name = Path(_input).name
if str(item) in config_files_list or str(
item) in code_files or str(item) in _outputs:
logging.info(f"Ignoring the config as an input: {item}")
else:
# Deleting the outputs of the inputs.
if item.is_dir():
if sorted([str(i)
for i in item.iterdir()]) == sorted(_outputs):
logging.info(f"Skipping: {item}")
else:
logging.info(
f"""Compressing the input directory ({name})""")
zip_file = compress_directory(
item, user_execution_directory)
dst_dir = data_dir
dst_file = dst_dir / Path(zip_file).name
if dst_file.exists():
os.remove(dst_file)
shutil.move(str(zip_file), str(dst_dir))
new_inputs.append(zip_file)
click.secho(f"""Input {name} added """, fg="blue")
logging.info("Input added: {}".format(name))
else:
new_inputs.append(item)
dst_file = mic_directory_path / DATA_DIR / str(item.name)
shutil.copy(item, dst_file)
click.secho(f"""Input {name} added """, fg="blue")
logging.info("Input added: {}".format(name))
info_end_inputs(new_inputs)
write_spec(mic_config_file, INPUTS_KEY,
relative(new_inputs, user_execution_directory))
write_spec(mic_config_file, CODE_KEY,
relative(code_files, user_execution_directory))
logging.info("inputs done")
except Exception as e:
logging.exception(f"Inputs failed: {e}")
click.secho("Failed", fg="red")
click.secho(e)
def add_parameters(mic_file, name, value, overwrite, description):
"""
Add a parameter into the MIC file (mic.yaml).
- You must pass the MIC file (mic.yaml) as an argument using the (-f) option; or run the command from the
same directory as the MIC file (mic.yaml)
Usage example:
mic pkg parameters -f <mic_file> --name PARAMETER_NAME --value PARAMETER_VALUE
"""
# Searches for mic file if user does not provide one
mic_file = check_mic_path(mic_file)
log_command(logging,
"add_parameters",
mic_file=mic_file,
name=name,
value=value,
overwrite=overwrite,
description=description)
try:
path = Path(mic_file)
spec = get_spec(path)
if (name or value or description or overwrite) and not (
(name is not None) and (value is not None)):
click.secho(
"Must give name and value to manually add new parameter. Aborting",
fg="yellow")
logging.info("Invalid manual parameter given")
exit(0)
if PARAMETERS_KEY not in spec:
spec[PARAMETERS_KEY] = {}
# Automacically add parameters from trace command. Use heuristic "if item isnt file its a parameter"
if name is None:
click.echo("Automatically adding any parameters from trace")
logging.info("Automatically adding any parameters from trace")
mic_config_file = Path(mic_file)
user_execution_directory = mic_config_file.parent.parent
repro_zip_trace_dir = find_dir(REPRO_ZIP_TRACE_DIR,
user_execution_directory)
repro_zip_trace_dir = Path(repro_zip_trace_dir)
repro_zip_config_file = repro_zip_trace_dir / REPRO_ZIP_CONFIG_FILE
reprozip_spec = get_spec(repro_zip_config_file)
spec = get_parameters_reprozip(spec, reprozip_spec)
else:
if not overwrite and name in spec[PARAMETERS_KEY]:
click.echo(
"The parameter exists. Add the option --overwrite to overwrite it."
)
logging.info(
"Parameter already exists. aborting because overwrite flag is false"
)
exit(1)
else:
if description is None:
description = ""
type_value____name__ = type(value).__name__
click.echo(
f"Adding the parameter {name}, value {value} and type {type_value____name__}"
)
new_par = {
name: {
NAME_KEY: name,
DEFAULT_VALUE_KEY: value,
DATATYPE_KEY: type_value____name__,
DEFAULT_DESCRIPTION_KEY: description
}
}
logging.debug("Adding parameter: {}".format(new_par))
spec[PARAMETERS_KEY].update(new_par)
write_spec(path, PARAMETERS_KEY, spec[PARAMETERS_KEY])
logging.info("add_parameters done")
except Exception as e:
logging.exception(f"Add Parameters failed: {e}")
click.secho("Failed", fg="red")
click.secho(e)
def trace(command, c, o):
"""
Complete the mic.yaml file with the information of the parameters and inputs you want to expose
MIC is going to automatically detect:
- All inputs (files and directories) used by your component and add them in the mic.yaml file.
- All parameters used by your component and add them in the configuration file
Usage example:
mic pkg trace python main.py
mic pkg trace ./your_program
"""
log_command(logging,
"trace",
invocation_command=command,
continu=c,
overwrite=o)
try:
if c and o:
click.secho(
"You can't use --continue and --overwrite at the same time",
fg="red")
logging.info("User tried to use -c and -o at same time")
exit(1)
append = None
if c:
append = True
if o:
append = False
logging.debug("Append mode: {}".format(append))
import reprozip.tracer.trace
import reprozip.traceutils
base_dir = REPRO_ZIP_TRACE_DIR
base = Path(".") / base_dir
output_reprozip = base / REPRO_ZIP_CONFIG_FILE
identify_packages = True
identify_inputs_outputs = True
now = datetime.now().timestamp()
status = reprozip.tracer.trace.trace(command[0], list(command),
base_dir, append, 1)
if status != 0:
click.secho("Program exited with non-zero code", fg="red")
logging.warning(
"Reprozip tracer exited with non-zero code: {}".format(status))
reprozip.tracer.trace.write_configuration(base,
identify_packages,
identify_inputs_outputs,
overwrite=False)
outputs = [
str(i.absolute()) for i in detect_new_reprozip(Path("."), now)
]
reprozip_spec = get_spec(output_reprozip)
reprozip_spec[OUTPUTS_KEY] = reprozip_spec[OUTPUTS_KEY].append(outputs) if OUTPUTS_KEY in reprozip_spec and \
reprozip_spec[
OUTPUTS_KEY] else outputs
write_to_yaml(output_reprozip, reprozip_spec)
logging.info("trace done")
except Exception as e:
logging.exception(f"Trace failed: {e}")
click.secho("Failed", fg="red")
click.secho(e)
def test_get_parameters():
yml = "example.yml"
spec = get_spec(Path(__file__).parent / RESOURCES / "cwl"/ yml)
parameters = get_parameters(spec)
assert len(parameters) == 8
def test_param_detection_v2():
mic_yaml = Path("254/mic/mic2.yaml")
mic_spec = get_spec(Path(__file__).parent / RESOURCES / mic_yaml)
repro_yaml = Path("254/.reprozip-trace2/config.yml")
repro_spec = get_spec(Path(__file__).parent / RESOURCES / repro_yaml)
spec = get_parameters_reprozip(mic_spec, repro_spec)
assert spec == {
"step": 1,
"name": "parameter-test",
"docker_image": "parameter-test:latest",
"inputs": {
"a_txt": {
"path": "a.txt",
"format": "txt"
},
"in_txt": {
"path": "in.txt",
"format": "txt"
}
},
"code_files": {
"addtoarray_sh": {
"path": "addtoarray.sh",
"format": "sh"
}
},
"outputs": {
"out_csv": {
"path": "outputs/out.csv",
"format": "csv"
}
},
"parameters": {
"param_1": {
"name": "",
"default_value": "15",
"type": "int",
"description": ""
},
"param_2": {
"name": "",
"default_value": "hello",
"type": "str",
"description": ""
},
"param_3": {
"name": "",
"default_value": "-3.1415",
"type": "float",
"description": ""
},
"param_4": {
"name": "",
"default_value": "-15",
"type": "int",
"description": ""
},
"param_5": {
"name": "",
"default_value": "test.txt",
"type": "str",
"description": ""
},
"param_6": {
"name": "",
"default_value": "string-special",
"type": "str",
"description": ""
}
}
}