def decode_run_json(input_json_file):
"""reads a run json file and creates three text files:
download command list file (commands to download input files from s3)
input yml file (for cwl/wdl/snakemake run)
env list file (environment variables to be sourced)
"""
# read json file
with open(input_json_file, 'r') as f:
runjson = AwsemRunJson(**json.load(f))
runjson_input = runjson.Job.Input
language = runjson.Job.App.language
# create a download command list file from the information in json
create_download_command_list(downloadlist_filename, runjson_input)
# create a bucket-mounting command list file
create_mount_command_list(mountlist_filename, runjson_input)
# create an input yml file to be used on awsem
if language in ['wdl', 'wdl_v1', 'wdl_draft2']: # wdl
create_input_for_wdl(input_yml_filename, runjson_input)
elif language == 'snakemake': # snakemake
create_input_for_snakemake(input_yml_filename, runjson_input)
else: # cwl
create_input_for_cwl(input_yml_filename, runjson_input)
# create a file that defines environmental variables
create_env_def_file(env_filename, runjson, language)
def test_create_env_def_file_snakemake():
"""testing create_env_def_file with shell option and two input Env variables"""
envfilename = 'someenvfile'
runjson_dict = {
'Job': {
'App': {
'language': 'snakemake',
'command': 'com1;com2',
'container_image': 'someimage',
'snakemake_url': 'someurl',
'main_snakemake': 'somesnakemake',
'other_snakemake_files': 'othersnakemake1,othersnakemake2'
},
'JOBID': 'somejobid',
'Input': {},
'Output': {
'output_bucket_directory': 'somebucket'
},
'JOBID': 'somejobid'
},
'config': {
'log_bucket': 'somebucket'
}
}
runjson = AwsemRunJson(**runjson_dict)
create_env_def_file(envfilename, runjson, 'snakemake')
with open(envfilename, 'r') as f:
envfile_content = f.read()
right_content = (
'export LANGUAGE=snakemake\n'
'export SNAKEMAKE_URL=someurl\n'
'export MAIN_SNAKEMAKE=somesnakemake\n'
'export SNAKEMAKE_FILES="othersnakemake1 othersnakemake2"\n'
'export COMMAND="com1;com2"\n'
'export CONTAINER_IMAGE=someimage\n'
'export PRESERVED_ENV_OPTION=""\n'
'export DOCKER_ENV_OPTION=""\n')
assert envfile_content == right_content
os.remove(envfilename)
def test_create_env_def_file_cwl():
"""testing create_env_def_file with cwl option and an input Env variable"""
envfilename = 'someenvfile'
runjson_dict = {
'Job': {
'App': {
'language': 'cwl_v1',
'cwl_url': 'someurl',
'main_cwl': 'somecwl',
'other_cwl_files': 'othercwl1,othercwl2'
},
'Input': {
'Env': {
'SOME_ENV': '1234'
}
},
'Output': {
'output_bucket_directory': 'somebucket'
},
'JOBID': 'somejobid'
},
'config': {
'log_bucket': 'somebucket'
}
}
runjson = AwsemRunJson(**runjson_dict)
create_env_def_file(envfilename, runjson, 'cwl')
with open(envfilename, 'r') as f:
envfile_content = f.read()
right_content = (
'export LANGUAGE=cwl_v1\n'
'export CWL_URL=someurl\n'
'export MAIN_CWL=somecwl\n'
'export CWL_FILES="othercwl1 othercwl2"\n'
'export SOME_ENV=1234\n'
'export PRESERVED_ENV_OPTION="--preserve-environment SOME_ENV "\n'
'export DOCKER_ENV_OPTION="-e SOME_ENV "\n')
assert envfile_content == right_content
os.remove(envfilename)
def test_create_env_def_file_shell():
"""testing create_env_def_file with shell option and two input Env variables"""
envfilename = 'someenvfile'
runjson_dict = {
'Job': {
'App': {
'language': 'shell',
'command': 'com1;com2',
'container_image': 'someimage'
},
'Input': {
'Env': {
'ENV1': '1234',
'ENV2': '5678'
}
},
'Output': {
'output_bucket_directory': 'somebucket'
},
'JOBID': 'somejobid'
},
'config': {
'log_bucket': 'somebucket'
}
}
runjson = AwsemRunJson(**runjson_dict)
create_env_def_file(envfilename, runjson, 'shell')
with open(envfilename, 'r') as f:
envfile_content = f.read()
right_content = (
'export LANGUAGE=shell\n'
'export COMMAND="com1;com2"\n'
'export CONTAINER_IMAGE=someimage\n'
'export ENV1=1234\n'
'export ENV2=5678\n'
'export PRESERVED_ENV_OPTION="--preserve-environment ENV1 --preserve-environment ENV2 "\n'
'export DOCKER_ENV_OPTION="-e ENV1 -e ENV2 "\n')
assert envfile_content == right_content
os.remove(envfilename)
def test_create_env_def_file_shell2():
"""testing create_env_def_file with shell option with complex commands and an env variable"""
envfilename = 'someenvfile'
complex_command = 'echo $SOME_ENV | xargs -i echo {} > somedir/somefile'
runjson_dict = {
'Job': {
'App': {
'language': 'shell',
'command': complex_command,
'container_image': 'someimage'
},
'Input': {
'Env': {
'SOME_ENV': '1234'
}
},
'Output': {
'output_bucket_directory': 'somebucket'
},
'JOBID': 'somejobid'
},
'config': {
'log_bucket': 'somebucket'
}
}
runjson = AwsemRunJson(**runjson_dict)
create_env_def_file(envfilename, runjson, 'shell')
with open(envfilename, 'r') as f:
envfile_content = f.read()
right_content = (
'export LANGUAGE=shell\n'
'export COMMAND="echo $SOME_ENV | xargs -i echo {} > somedir/somefile"\n'
'export CONTAINER_IMAGE=someimage\n'
'export SOME_ENV=1234\n'
'export PRESERVED_ENV_OPTION="--preserve-environment SOME_ENV "\n'
'export DOCKER_ENV_OPTION="-e SOME_ENV "\n')
assert envfile_content == right_content
os.remove(envfilename)
def test_create_env_def_file_shell3():
"""testing create_env_def_file with shell option with complex commands and an env variable.
double-quotes are escaped when written to the env file ('"' -> '\"')"""
envfilename = 'someenvfile'
complex_command = 'echo "haha" > somefile; ls -1 [st]*'
runjson_dict = {
'Job': {
'App': {
'language': 'shell',
'command': complex_command,
'container_image': 'someimage'
},
'Input': {
'Env': {}
},
'Output': {
'output_bucket_directory': 'somebucket'
},
'JOBID': 'somejobid'
},
'config': {
'log_bucket': 'somebucket'
}
}
runjson = AwsemRunJson(**runjson_dict)
create_env_def_file(envfilename, runjson, 'shell')
with open(envfilename, 'r') as f:
envfile_content = f.read()
right_content = (
'export LANGUAGE=shell\n'
'export COMMAND="echo \\"haha\\" > somefile; ls -1 [st]*"\n'
'export CONTAINER_IMAGE=someimage\n'
'export PRESERVED_ENV_OPTION=""\n'
'export DOCKER_ENV_OPTION=""\n')
assert envfile_content == right_content
os.remove(envfilename)
def test_create_env_def_file_wdl_v1():
"""testing create_env_def_file with wdl option and no input Env variable"""
envfilename = 'someenvfile'
runjson_dict = {
'Job': {
'App': {
'language': 'wdl_v1',
'wdl_url': 'someurl',
'main_wdl': 'somewdl',
'other_wdl_files': 'otherwdl1,otherwdl2'
},
'Input': {
'Env': {}
},
'Output': {
'output_bucket_directory': 'somebucket'
},
'JOBID': 'somejobid'
},
'config': {
'log_bucket': 'somebucket'
}
}
runjson = AwsemRunJson(**runjson_dict)
create_env_def_file(envfilename, runjson, 'wdl_v1')
with open(envfilename, 'r') as f:
envfile_content = f.read()
right_content = ('export LANGUAGE=wdl_v1\n'
'export WDL_URL=someurl\n'
'export MAIN_WDL=somewdl\n'
'export WDL_FILES="otherwdl1 otherwdl2"\n'
'export PRESERVED_ENV_OPTION=""\n'
'export DOCKER_ENV_OPTION=""\n')
assert envfile_content == right_content
os.remove(envfilename)