def _create_job(task, cluster, folder, image, container):
params = _get_job_parameters(task, cluster, image, container)
setup_commands = params['setupCommands']
repository = params['repository']
tag = params['tag']
job_parameters = params['jobParameters']
run_command = 'python pull.py -r %s -t %s -c %s' % (repository, tag,
container)
commands = setup_commands + [run_command, 'rm pull.py']
body = {
# ensure there are no special characters in the submission script name
'name': 'desc_%s' % re.sub('[^a-zA-Z0-9]', '_', run_command),
'commands': commands,
'input': [{
'folderId': folder['_id'],
'path': '.'
}],
'output': [{
'folderId': folder['_id'],
'path': '.'
}],
'uploadOutput': False,
'params': job_parameters
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
return job
def create_export_job(task, job_name, files, job_dir, mesh_filename):
commands = []
mesh_file_path = os.path.join(job_dir, 'input', mesh_filename)
for file in files:
name = file['name']
vtk_filename = '%s.vtu' % name.rsplit('.', 1)[0]
output_path = os.path.join(job_dir, vtk_filename)
solution_file_path = os.path.join(job_dir, name)
cmd = 'pyfr export %s %s %s' % (mesh_file_path, solution_file_path,
output_path)
commands.append(cmd)
body = {
'name': job_name,
'commands': commands,
'input': [],
'output': [],
'params': {
'numberOfSlots': 1
}
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.logger.info('Created export job %s' % job['_id'])
return job
def _create_job_nersc(task, cluster, input_file, input_folder):
task.taskflow.logger.info('Create NWChem job.')
body = {
'name':
'nwchem_run',
'commands': [
'/usr/bin/srun -N 1 -n 32 %s %s' %
(os.environ.get('OC_NWCHEM_PATH', 'nwchem'), input_file['name'])
],
'input': [{
'folderId': input_folder['_id'],
'path': '.'
}],
'output': [],
'params': {
'taskFlowId': task.taskflow.id,
'numberOfNodes': 1,
'queue': 'debug',
'constraint': 'haswell',
'account': os.environ.get('OC_ACCOUNT')
}
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.taskflow.set_metadata('jobs', [job])
return job
def create_export_job(task, job_name, files, job_dir, mesh_filename):
commands = []
mesh_file_path = os.path.join(job_dir, 'input', mesh_filename)
for file in files:
name = file['name']
vtk_filename = '%s.vtu' % name.rsplit('.', 1)[0]
output_path = os.path.join(job_dir, vtk_filename)
solution_file_path = os.path.join(job_dir, name)
cmd = 'pyfr export %s %s %s' % (mesh_file_path,
solution_file_path, output_path)
commands.append(cmd)
body = {
'name': job_name,
'commands': commands,
'input': [],
'output': [],
'params': {
'numberOfSlots': 1
}
}
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.logger.info('Created export job %s' % job['_id'])
return job
def upload_input(task, cluster, job, *args, **kwargs):
file_id = parse('input.file.id').find(kwargs)
if file_id:
file_id = file_id[0].value
task.logger.info('Visualizing file ID: %s' % file_id)
job['params']['dataDir'] = '$PWD'
# Fetch the file
girder_client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
file = girder_client.getResource('file', file_id)
# Set the file to load
filename = file['name']
job['params']['fileName'] = filename
task.logger.info('Filename is: %s' % filename)
# Set main script if not visualizer
if MAIN_SCRIPT:
job['params']['scriptToRun'] = MAIN_SCRIPT
task.logger.info('Swap run script to: %s' % MAIN_SCRIPT)
task.logger.info('Uploading file to cluster.')
job_dir = job_directory(cluster, job)
upload_file(cluster, task.taskflow.girder_token, file, job_dir)
task.logger.info('Upload complete.')
def create_job(task, *args, **kwargs):
task.logger.info('Taskflow %s' % task.taskflow.id)
task.taskflow.logger.info('Create PyFr job.')
input_folder_id = kwargs['input']['folder']['id']
backend = kwargs['backend']['type']
body = {
'name':
'pyfr_run',
'commands': [
"mpiexec -n %s pyfr run -b %s input/%s input/%s" %
(kwargs['numberOfProcs'], backend, kwargs['meshFilename'],
kwargs['iniFilename'])
],
'input': [{
'folderId': input_folder_id,
'path': 'input'
}],
'output': [],
'params': {
'numberOfSlots': kwargs['numberOfProcs']
}
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.taskflow.set_metadata('jobs', [job])
submit.delay(job, *args, **kwargs)
def export_output(task, folder_id, imported_mesh_file_id, files):
"""
Export a batch of PyFR solution files into VTK format.
:param: folder_id: The target folder id to upload the VTK files to.
:param: imported_mesh_file_id: The mesh in PyFR format
:param: files: The files to export ( Girder JSON objects )
"""
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
output_dir = tempfile.mkdtemp()
try:
mesh_path = os.path.join(output_dir, 'mesh.pyfrm')
task.logger.info('Downloading mesh.')
client.downloadFile(imported_mesh_file_id, mesh_path)
task.logger.info('Downloading complete.')
for file in files:
_export_file(task, client, folder_id, mesh_path, file, output_dir)
finally:
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
def export_output(task, folder_id, imported_mesh_file_id, files):
"""
Export a batch of PyFR solution files into VTK format.
:param: folder_id: The target folder id to upload the VTK files to.
:param: imported_mesh_file_id: The mesh in PyFR format
:param: files: The files to export ( Girder JSON objects )
"""
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
output_dir = tempfile.mkdtemp()
try:
mesh_path = os.path.join(output_dir, 'mesh.pyfrm')
task.logger.info('Downloading mesh.')
client.downloadFile(imported_mesh_file_id, mesh_path)
task.logger.info('Downloading complete.')
for file in files:
_export_file(task, client, folder_id, mesh_path, file, output_dir)
finally:
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
def create_job(task, upstream_result):
task.logger.info('Taskflow %s' % task.taskflow.id)
task.taskflow.logger.info('Create NWChem job.')
input_folder_id = upstream_result['input']['folder']['id']
# TODO: setup command to run with mpi
body = {
'name': 'nwchem_run',
'commands': [
"mpiexec -n %s nwchem input/%s" % (
upstream_result['numberOfProcs'],
upstream_result['nwFilename'])
],
'input': [
{
'folderId': input_folder_id,
'path': 'input'
}
],
'output': [],
'params': {
'numberOfSlots': upstream_result['numberOfProcs']
}
}
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
upstream_result['job'] = job
task.taskflow.set_metadata('jobs', [job])
return upstream_result
def _create_job_ec2(task, cluster, input_file, input_folder):
task.taskflow.logger.info('Create NWChem job.')
input_name = input_file['name']
body = {
'name':
'nwchem_run',
'commands': [
'docker pull openchemistry/nwchem-json:latest',
'docker run -v $(pwd):/data openchemistry/nwchem-json:latest %s' %
(input_name)
],
'input': [{
'folderId': input_folder['_id'],
'path': '.'
}],
'output': [],
'params': {
'taskFlowId': task.taskflow.id
}
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.taskflow.set_metadata('jobs', [job])
return job
def upload_output(task, cluster, job, *args, **kwargs):
# Girder client
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
# Refresh state of job
job = client.get('jobs/%s' % job['_id'])
# Register generated file into girder
task.taskflow.logger.info('Uploading results from cluster')
output_folder_id = parse('output.folder.id').find(kwargs)
if output_folder_id:
task.taskflow.logger.info('Exporting to folder %s' % output_folder_id)
output_folder_id = output_folder_id[0].value
job['output'] = [{'folderId': output_folder_id, 'path': '.'}]
else:
task.taskflow.logger.info('No output.folder.id')
# Upload files metadata
upload_job_output_to_folder(cluster,
job,
log_write_url=None,
job_dir=None,
girder_token=task.taskflow.girder_token)
# Done...
task.taskflow.logger.info('Upload complete.')
def create_paraview_job(task, *args, **kwargs):
_update_cluster_config(task, kwargs['cluster'])
task.logger.info('Validating args passed to flow.')
validate_args(kwargs)
cluster = kwargs.pop('cluster')
# Save the cluster in the taskflow for termination
task.taskflow.set_metadata('cluster', cluster)
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
task.taskflow.logger.info('Creating ParaView job.')
task.logger.info('Load ParaView submission script.')
base_path = os.path.dirname(__file__)
script_path = os.path.join(base_path, 'pvw.sh')
if not os.path.exists(script_path):
msg = 'Script path %s does not exists.' % script_path
task.logger.info(msg)
raise Exception(msg)
with open(script_path, 'r') as fp:
commands = fp.read().splitlines()
body = {
'name': 'paraview',
'commands': commands,
'input': [],
'output': []
}
job = client.post('jobs', data=json.dumps(body))
task.logger.info('ParaView job created: %s' % job['_id'])
task.taskflow.logger.info('ParaView job created.')
task.taskflow.set_metadata('jobs', [job])
# Upload the visualizer code
task.logger.info('Uploading visualizer')
viz_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../../../../',
'node_modules/pvw-visualizer/server/pvw-visualizer.py'))
if not os.path.exists(viz_path):
task.logger.error(
'Unable to locate pvw-visualizer.py for upload. (%s)' % viz_path)
return
target_dir = job_directory(cluster, job)
target_path = os.path.join(target_dir, 'pvw-visualizer.py')
with get_connection(task.taskflow.girder_token, cluster) as conn:
conn.makedirs(target_dir)
with open(viz_path, 'r') as fp:
conn.put(fp, target_path)
submit_paraview_job.delay(cluster, job, *args, **kwargs)
def cleanup_proxy_entries(task):
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
session_key = parse('meta.sessionKey').find(task.taskflow)
if session_key:
session_key = session_key[0].value
client.delete('proxy/%s' % session_key)
def cleanup_proxy_entries(task):
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
session_key = parse('meta.sessionKey').find(task.taskflow)
if session_key:
session_key = session_key[0].value
client.delete('proxy/%s' % session_key)
def create_paraview_job(task, *args, **kwargs):
_update_cluster_config(task, kwargs['cluster'])
task.logger.info('Validating args passed to flow.')
validate_args(kwargs)
cluster = kwargs.pop('cluster')
# Save the cluster in the taskflow for termination
task.taskflow.set_metadata('cluster', cluster)
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
task.taskflow.logger.info('Creating ParaView job.')
task.logger.info('Load ParaView submission script.')
base_path = os.path.dirname(__file__)
script_path = os.path.join(base_path, 'pvw.sh')
if not os.path.exists(script_path):
msg = 'Script path %s does not exists.' % script_path
task.logger.info(msg)
raise Exception(msg)
with open(script_path, 'r') as fp:
commands = fp.read().splitlines()
body = {
'name': 'paraview',
'commands': commands,
'input': [],
'output': []
}
job = client.post('jobs', data=json.dumps(body))
task.logger.info('ParaView job created: %s' % job['_id'])
task.taskflow.logger.info('ParaView job created.')
task.taskflow.set_metadata('jobs', [job])
# Upload the visualizer code
task.logger.info('Uploading visualizer')
viz_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../', '../', '../','../',
'node_modules/pvw-visualizer/server/pvw-visualizer.py'))
if not os.path.exists(viz_path):
task.logger.error('Unable to local pvw-visualizer.py for upload.')
return
target_dir = job_directory(cluster, job)
target_path = os.path.join(target_dir, 'pvw-visualizer.py')
with get_connection(task.taskflow.girder_token, cluster) as conn:
conn.makedirs(target_dir)
with open(viz_path, 'r') as fp:
conn.put(fp, target_path)
submit_paraview_job.delay(cluster, job, *args, **kwargs)
def create_openfoam_job(task, *args, **kwargs):
# Girder client
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
# Save the cluster in the taskflow for termination
cluster = kwargs.pop('cluster')
task.taskflow.set_metadata('cluster', cluster)
# Create job definition
task.taskflow.logger.info('Creating OpenFoam job.')
body = {
'name':
'openfoam_run',
'commands': [
# TODO: Setting command for Visualizer's access
'chmod +x $PWD/run.sh',
# NOTE: idk what the command below
#'xhost +local:of_v1612_plus',
#'docker start of_v1612_plus',
'$PWD/run.sh $PWD'
],
'input': [{
'folderId': kwargs['input']['folder']['id'],
'path': '.'
}],
'output': [{
'path': 'simulation/log.blockMesh'
}, {
'path': 'simulation/log.patchSummary'
}, {
'path': 'simulation/log.potentialFoam'
}, {
'path': 'simulation/log.reconstructParMesh'
}, {
'path': 'simulation/log.surfaceFeatureExtract'
}, {
'path': 'simulation/log.snappyHexMesh'
}, {
'path': 'simulation/log.simpleFoam'
}]
}
# Register job in girder + attach to taskflow
job = client.post('jobs', data=json.dumps(body))
task.logger.info('OpenFOAM job created: %s' % job['_id'])
task.taskflow.logger.info('OpenFOAM job created.')
task.taskflow.set_metadata('jobs', [job])
# Capture job working directory
target_dir = job_directory(cluster, job)
task.taskflow.set_metadata('dataDir', target_dir)
# Move to the next task
submit_open_foam_job.delay(cluster, job, *args, **kwargs)
def create_openfoam_job(task, *args, **kwargs):
# Girder client
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
# Save the cluster in the taskflow for termination
cluster = kwargs.pop('cluster')
task.taskflow.set_metadata('cluster', cluster)
# Create job definition
task.taskflow.logger.info('Creating OpenFoam job.')
body = {
'name': 'openfoam_run',
'commands': [
'python $PWD/simput-unpack.py $PWD/input-deck.json $PWD',
'docker start of_v1612_plus',
'docker exec -t of_v1612_plus $PWD/DockerRun $PWD'
],
'input': [
{
'folderId': kwargs['input']['folder']['id'],
'path': '.'
},
{
'folderId': kwargs['input']['project']['folder']['id'],
'path': '.'
}
],
'output': [
]
}
# Register job in girder + attach to taskflow
job = client.post('jobs', data=json.dumps(body))
task.logger.info('OpenFOAM job created: %s' % job['_id'])
task.taskflow.logger.info('OpenFOAM job created.')
task.taskflow.set_metadata('jobs', [job])
# Capture job working directory
target_dir = job_directory(cluster, job)
task.taskflow.set_metadata('dataDir', target_dir)
source_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../../../../',
'node_modules/simput/bin/unpack/simput-unpack.py'))
target_path = os.path.join(target_dir, 'simput-unpack.py')
# Upload unpack script
with get_connection(task.taskflow.girder_token, cluster) as conn:
conn.makedirs(target_dir)
with open(source_path, 'r') as fp:
conn.put(fp, target_path)
# Move to the next task
submit_open_foam_job.delay(cluster, job, *args, **kwargs)
def _get_digest(task, repository, tag):
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
params = {'repository': repository, 'tag': tag}
r = client.get('images', params)
images = r['results']
if len(images) < 1:
log_and_raise(task, 'Image not found on the server.')
# The digest should definitely be here
return images[0]['digest']
def create_proxy_entry(task, cluster, job):
session_key = job['params']['sessionKey']
host = cluster['config']['host']
body = {
'host': host,
'port': cluster['config']['paraview']['websocketPort'],
'key': session_key
}
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
client.post('proxy', data=json.dumps(body))
def create_proxy_entry(task, cluster, job):
session_key = job['params']['sessionKey']
host = cluster['config']['host']
body = {
'host': host,
'port': cluster['config']['paraview']['websocketPort'],
'key': session_key
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
client.post('proxy', data=json.dumps(body))
def _ensure_image_on_server(task, repository, tag, digest, container='docker'):
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
params = {'repository': repository, 'tag': tag, 'digest': digest}
r = client.get('images', params)
images = r['results']
if len(images) < 1:
log_and_raise(task, 'Image not found on the server.')
if container not in images[0]:
msg = 'Image does not have container type: ' + container
log_and_raise(task, msg)
def postprocess_job(task, _, input_, cluster, image, run_parameters,
root_folder, container_description, input_folder,
output_folder, scratch_folder, job):
task.taskflow.logger.info('Processing the results of the job.')
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
# Refresh state of job
job = client.get('jobs/%s' % job['_id'])
upload_job_output_to_folder(cluster,
job,
girder_token=task.taskflow.girder_token)
# remove temporary input folder folder, this data is attached to the calculation model
client.delete('folder/%s' % input_folder['_id'])
# clean up the scratch folder
keep_scratch = run_parameters.get('keepScratch', False)
if keep_scratch:
scratch_folder_id = scratch_folder['_id']
else:
client.delete('folder/%s' % scratch_folder['_id'])
scratch_folder_id = None
# ingest the output of the calculation
output_format = container_description['output']['format']
output_file = None
output_items = list(client.listItem(output_folder['_id']))
for item in output_items:
if item['name'] == 'output.%s' % output_format:
files = list(client.listFile(item['_id']))
if len(files) != 1:
raise Exception(
'Expecting a single file under item, found: %s' +
len(files))
output_file = files[0]
break
if output_file is None:
raise Exception('The calculation did not produce any output file.')
# Now call endpoint to ingest result
body = {
'fileId': output_file['_id'],
'format': output_format,
'public': True,
'image':
image, # image now also has a digest field, add it to the calculation
'scratchFolderId': scratch_folder_id
}
client.put('calculations/%s' % input_['calculation']['_id'], json=body)
def paraview_terminate(task):
cluster = parse('meta.cluster').find(task.taskflow)
if cluster:
cluster = cluster[0].value
else:
task.logger.warning('Unable to extract cluster from taskflow. '
'Unable to terminate ParaView job.')
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
jobs = task.taskflow.get('meta', {}).get('jobs', [])
terminate_jobs(task, client, cluster, jobs)
def paraview_terminate(task):
cluster = parse('meta.cluster').find(task.taskflow)
if cluster:
cluster = cluster[0].value
else:
task.logger.warning('Unable to extract cluster from taskflow. '
'Unable to terminate ParaView job.')
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
jobs = task.taskflow.get('meta', {}).get('jobs', [])
terminate_jobs(task, client, cluster, jobs)
def _create_description_job(task, cluster, description_folder, image,
run_parameters):
params = _get_job_parameters(task, cluster, image, run_parameters)
container = params['container']
setup_commands = params['setupCommands']
repository = params['repository']
tag = params['tag']
digest = params['digest']
job_parameters = params['jobParameters']
# Make sure the image exists on the server before we go further
_ensure_image_on_server(task, repository, tag, digest, container)
output_file = 'description.json'
run_command = '%s run $IMAGE_NAME' % container
# Shifter has a pretty different sytax so special case it.
if container == 'shifter':
run_command = 'shifter --image=$IMAGE_NAME --entrypoint --'
image_name = '%s:%s' % (repository, tag)
if container == 'singularity':
image_name = digest_to_sif(digest)
commands = setup_commands + [
'IMAGE_NAME=%s' % image_name,
'%s -d > %s' % (run_command, output_file)
]
body = {
# ensure there are no special characters in the submission script name
'name': 'desc_%s' % re.sub('[^a-zA-Z0-9]', '_', repository),
'commands': commands,
'input': [{
'folderId': description_folder['_id'],
'path': '.'
}],
'output': [{
'folderId': description_folder['_id'],
'path': '.'
}],
'uploadOutput': False,
'params': job_parameters
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
return job
def postprocess_job(task, _, user, cluster, image, job, folder, container):
task.taskflow.logger.info('Finished pulling the container')
task.taskflow.logger.info('Processing the results of the pull.')
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
# Refresh state of job
job = client.get('jobs/%s' % job['_id'])
upload_job_output_to_folder(cluster,
job,
girder_token=task.taskflow.girder_token)
output_items = list(client.listItem(folder['_id']))
output_file = None
for item in output_items:
if item['name'] == OUTPUT_FILE:
files = list(client.listFile(item['_id']))
if len(files) != 1:
log_and_raise(
task, 'Expecting a single file under item, found: %s' %
len(files))
output_file = files[0]
break
if output_file is None:
log_and_raise(task, 'Could not locate output file: %s' % OUTPUT_FILE)
# Download the output file
output_io = io.BytesIO()
client.downloadFile(output_file['_id'], output_io)
# Remove the folder
client.delete('folder/%s' % folder['_id'])
pull_json = json.loads(output_io.getvalue().decode('utf-8'))
image_uri = pull_json.get('imageUri')
# Convert size to GB
size = round(pull_json.get('size', 0) / 1.e9, 2)
_ensure_image_uri_is_valid(task, container, image_uri)
repository = image.get('repository')
tag = image.get('tag')
digest = _extract_digest(container, image_uri)
post_image_to_database(client, container, repository, tag, digest, cluster,
size)
task.taskflow.logger.info('Success!')
def start(task, input_, user, cluster, image, run_parameters):
"""
The flow is the following:
- Dry run the container with the -d flag to obtain a description of the input/output formats
- Convert the cjson input geometry to conform to the container's expected format
- Run the container
- Convert the container output format into cjson
- Ingest the output in the database
"""
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
if cluster.get('name') == 'cori':
cluster = get_cori(client)
if '_id' not in cluster:
log_and_raise(task, 'Invalid cluster configurations: %s' % cluster)
oc_folder = get_oc_folder(client)
root_folder = client.createFolder(
oc_folder['_id'],
datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%f"))
# temporary folder to save the container in/out description
description_folder = client.createFolder(root_folder['_id'], 'description')
job = _create_description_job(task, cluster, description_folder, image,
run_parameters)
task.taskflow.logger.info(
'Preparing job to obtain the container description.')
download_job_input_folders(cluster,
job,
girder_token=task.taskflow.girder_token,
submit=False)
task.taskflow.logger.info(
'Submitting job to obtain the container description.')
submit_job(cluster,
job,
girder_token=task.taskflow.girder_token,
monitor=False)
monitor_job.apply_async(
(cluster, job), {
'girder_token': task.taskflow.girder_token,
'monitor_interval': 10
},
countdown=countdown(cluster),
link=postprocess_description.s(input_, user, cluster, image,
run_parameters, root_folder, job,
description_folder))
def _update_cluster_config(task, cluster):
if cluster['type'] == 'ec2':
paraview_config = cluster['config'].setdefault('paraview', {})
paraview_config['installDir'] = '/opt/paraview'
paraview_config['websocketPort'] = 9000
# Update ParaView config on cluster
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
client.patch('clusters/%s' % cluster['_id'],
data=json.dumps({
'config': {
'paraview': cluster['config']['paraview']
}
}))
def _update_cluster_config(task, cluster):
if cluster['type'] == 'ec2':
paraview_config = cluster['config'].setdefault('paraview', {})
paraview_config['installDir'] = '/opt/paraview'
paraview_config['websocketPort'] = 9000
# Update ParaView config on cluster
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
client.patch('clusters/%s' % cluster['_id'],
data=json.dumps({
'config': {
'paraview': cluster['config']['paraview']
}
}))
def start(task, user, cluster, image, container):
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
if cluster.get('name') == 'cori':
cluster = get_cori(client)
if '_id' not in cluster:
log_and_raise(task, 'Invalid cluster configurations: %s' % cluster)
oc_folder = get_oc_folder(client)
root_folder = client.createFolder(
oc_folder['_id'],
datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%f"))
# temporary folder to save the container in/out description
folder = client.createFolder(root_folder['_id'], 'pull_folder')
# save the pull.py script to the job directory
with open(os.path.join(os.path.dirname(__file__), 'utils/pull.py'),
'rb') as f:
# Get the size of the file
size = f.seek(0, 2)
f.seek(0)
name = 'pull.py'
client.uploadFile(folder['_id'], f, name, size, parentType='folder')
job = _create_job(task, cluster, folder, image, container)
# Now download pull.py script to the cluster
task.taskflow.logger.info('Preparing job to pull the container.')
download_job_input_folders(cluster,
job,
girder_token=task.taskflow.girder_token,
submit=False)
task.taskflow.logger.info('Submitting job to pull the container.')
submit_job(cluster,
job,
girder_token=task.taskflow.girder_token,
monitor=False)
monitor_job.apply_async((cluster, job), {
'girder_token': task.taskflow.girder_token,
'monitor_interval': 10
},
countdown=countdown(cluster),
link=postprocess_job.s(user, cluster, image, job,
folder, container))
def upload_output(task, cluster, job, *args, **kwargs):
task.taskflow.logger.info('Uploading results from cluster')
# Refresh state of job
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
job = client.get('jobs/%s' % job['_id'])
output_folder_id = parse('output.folder.id').find(kwargs)
if output_folder_id:
output_folder_id = output_folder_id[0].value
job['output'] = [{
'folderId': output_folder_id,
'path': '.'
}]
upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None,
girder_token=task.taskflow.girder_token)
task.taskflow.logger.info('Upload complete.')
def upload_output(task, cluster, job, *args, **kwargs):
task.taskflow.logger.info('Uploading results from cluster')
# Refresh state of job
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.get('jobs/%s' % job['_id'])
output_folder_id = parse('output.folder.id').find(kwargs)
if output_folder_id:
output_folder_id = output_folder_id[0].value
job['output'] = [{'folderId': output_folder_id, 'path': '.'}]
upload_job_output_to_folder(cluster,
job,
log_write_url=None,
job_dir=None,
girder_token=task.taskflow.girder_token)
task.taskflow.logger.info('Upload complete.')
def upload_input(task, cluster, job, *args, **kwargs):
file_id = parse('input.file.id').find(kwargs)
if file_id:
file_id = file_id[0].value
task.logger.info('Visualizing file ID: %s' % file_id)
job['params']['dataDir'] = '.'
# Fetch the file
girder_client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
file = girder_client.getResource('file', file_id)
# Set the file to load
filename = file['name']
job['params']['fileName'] = filename
task.logger.info('Filename is: %s' % filename)
task.logger.info('Uploading file to cluster.')
job_dir = job_directory(cluster, job)
upload_file(cluster, task.taskflow.girder_token, file, job_dir)
task.logger.info('Upload complete.')
def upload_export_output(task, _, cluster, job, *args, **kwargs):
output_folder_id = kwargs['output']['folder']['id']
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
for job_id in task.taskflow.get_metadata('export_jobs')['export_jobs']:
# Get job
export_job = client.get('jobs/%s' % job_id)
export_job['output'] = [{
'folderId': output_folder_id,
'path': '.'
}]
upload_job_output_to_folder(cluster, export_job, log_write_url=None,
job_dir=None, girder_token=task.taskflow.girder_token)
# Upload the vtu files
girder_token = task.taskflow.girder_token
download_path_from_cluster(cluster, girder_token, output_folder_id, job['dir'],
include=['^.*\\.vtu$'])
def upload_export_output(task, _, cluster, job, *args, **kwargs):
output_folder_id = kwargs['output']['folder']['id']
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
for job_id in task.taskflow.get_metadata('export_jobs')['export_jobs']:
# Get job
export_job = client.get('jobs/%s' % job_id)
export_job['output'] = [{
'folderId': output_folder_id,
'path': '.'
}]
upload_job_output_to_folder(cluster, export_job, log_write_url=None,
job_dir=None, girder_token=task.taskflow.girder_token)
# Upload the vtu files
girder_token = task.taskflow.girder_token
download_path_from_cluster(cluster, girder_token, output_folder_id, job['dir'],
include=['^.*\\.vtu$'])
def _create_description_job(task, cluster, description_folder, image,
run_parameters):
params = _get_job_parameters(cluster, image, run_parameters)
container = params['container']
setup_commands = params['setupCommands']
repository = params['repository']
tag = params['tag']
job_params = {'taskFlowId': task.taskflow.id}
output_file = 'description.json'
commands = setup_commands + [
'IMAGE_NAME=$(python pull.py -r %s -t %s -c %s | tail -1)' %
(repository, tag, container),
'%s run $IMAGE_NAME -d > %s' % (container, output_file), 'rm pull.py'
]
body = {
# ensure there are no special characters in the submission script name
'name': 'desc_%s' % re.sub('[^a-zA-Z0-9]', '_', repository),
'commands': commands,
'input': [{
'folderId': description_folder['_id'],
'path': '.'
}],
'output': [{
'folderId': description_folder['_id'],
'path': '.'
}],
'uploadOutput': False,
'params': job_params
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
return job
def upload_output(task, upstream_result):
task.taskflow.logger.info('Uploading results from cluster')
output_folder_id = upstream_result['output']['folder']['id']
cluster = upstream_result['cluster']
job = upstream_result['job']
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
# Refresh state of job
job = client.get('jobs/%s' % job['_id'])
job['output'] = [{
'folderId': output_folder_id,
'path': '.'
}]
upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None,
girder_token=task.taskflow.girder_token)
task.taskflow.logger.info('Upload job output complete.')
return upstream_result
def setup_input(task, *args, **kwargs):
input_folder_id = kwargs['input']['folder']['id']
number_of_procs = kwargs.get('numberOfSlots')
if not number_of_procs:
number_of_procs = kwargs.get('numberOfNodes')
if not number_of_procs:
size = parse('cluster.config.launch.params.node_instance_count').find(kwargs)
if size:
number_of_procs = size[0].value + 1
else:
raise Exception('Unable to extract number of nodes in cluster')
if not number_of_procs:
raise Exception('Unable to determine number of mpi processes to run.')
number_of_procs = int(number_of_procs)
kwargs['numberOfProcs'] = number_of_procs
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
geometry_file_id = parse('input.geometryFile.id').find(kwargs)
if geometry_file_id:
geometry_file_id = geometry_file_id[0].value
kwargs['geometryFileId'] = geometry_file_id
# Get the geometry file metadata to see if we need to import
geometry_file = client.getResource('file/%s' % geometry_file_id)
kwargs['geometryFilename'] = geometry_file['name']
ini_file_id = kwargs['input']['nwFile']['id']
ini_file = client.getResource('file/%s' % ini_file_id)
kwargs['nwFilename'] = ini_file['name']
return kwargs
def postprocess(task, _, run_folder, input_, cluster, job):
task.taskflow.logger.info('Uploading results from cluster')
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
output_folder = client.createFolder(run_folder['_id'], 'output')
# Refresh state of job
job = client.get('jobs/%s' % job['_id'])
job['output'] = [{'folderId': output_folder['_id'], 'path': '.'}]
upload_job_output_to_folder(cluster,
job,
girder_token=task.taskflow.girder_token)
task.taskflow.logger.info('Upload job output complete.')
input_file_name = task.taskflow.get_metadata('inputFileName')
input_file_name
# Call to ingest the files
for item in client.listItem(output_folder['_id']):
if item['name'].endswith('.json'):
files = list(client.listFile(item['_id']))
if len(files) != 1:
raise Exception(
'Expecting a single file under item, found: %s' +
len(files))
json_output_file_id = files[0]['_id']
# Now call endpoint to ingest result
body = {
'calculationId': input_['calculation']['_id'],
'fileId': json_output_file_id,
'public': True
}
client.post('molecules', json=body)
def create_job(task, *args, **kwargs):
task.logger.info('Taskflow %s' % task.taskflow.id)
task.taskflow.logger.info('Create PyFr job.')
input_folder_id = kwargs['input']['folder']['id']
backend = kwargs['backend']['type']
body = {
'name': 'pyfr_run',
'commands': [
"mpiexec -n %s pyfr run -b %s input/%s input/%s" % (
kwargs['numberOfProcs'],
backend,
kwargs['meshFilename'],
kwargs['iniFilename'])
],
'input': [
{
'folderId': input_folder_id,
'path': 'input'
}
],
'output': [],
'params': {
'numberOfSlots': kwargs['numberOfProcs']
}
}
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.taskflow.set_metadata('jobs', [job])
submit.delay(job, *args, **kwargs)
def pyfr_terminate(task):
cluster = task.taskflow['meta']['cluster']
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
terminate_jobs(
task, client, cluster, task.taskflow.get('meta', {}).get('jobs', []))
def upload_output(task, _, cluster, job, *args, **kwargs):
task.taskflow.logger.info('Uploading results from cluster')
output_folder_id = kwargs['output']['folder']['id']
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
# Refresh state of job
job = client.get('jobs/%s' % job['_id'])
job['output'] = [{
'folderId': output_folder_id,
'path': '.'
}]
upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None,
girder_token=task.taskflow.girder_token)
task.taskflow.logger.info('Upload job output complete.')
mesh_file_id = kwargs.pop('meshFileId')
solution_files = list(_list_solution_files(client, output_folder_id))
if len(solution_files) == 0:
raise Exception('No solution files where produced, please check output files for errors.')
# Generate and save the first vtu file that should be loaded for this
# run. This can then be used to know which file to open as part of any viz
# step.
file_names = [f['name'] for f in solution_files]
file_names.sort()
vtu_file = '%s.vtu' % file_names[0].rsplit('.', 1)[0]
task.taskflow.set_metadata('vtuFile', vtu_file)
number_files = len(solution_files)
# By default export solution files to VTK format using a set of batch jobs
if not 'exportInTaskFlow' in kwargs or not kwargs['exportInTaskFlow']:
number_of_jobs = kwargs['numberOfProcs']
task.logger.info('Generating %d export jobs' % number_of_jobs)
sim_job_dir = job['dir']
jobs = []
job_index = 1
for chunk in [solution_files[i::number_of_jobs] for i in xrange(number_of_jobs)]:
if chunk:
name = 'pyfr_export_%d' % job_index
mesh_filename = kwargs['meshFilename']
export_job = create_export_job(
task, name, chunk, sim_job_dir, mesh_filename)
submit_job(cluster, export_job, log_write_url=None,
girder_token=task.taskflow.girder_token, monitor=False)
jobs.append(export_job)
job_index += 1
# Update the jobs list in the metadata
task.taskflow.set_metadata('jobs', [j for j in jobs] +
[job])
# Also save just the export job ids
task.taskflow.set_metadata('export_jobs', [j['_id'] for j in jobs])
monitor_jobs.apply_async(
(cluster, jobs), {'girder_token': task.taskflow.girder_token},
link=upload_export_output.s(cluster, job, *args, **kwargs))
else:
# The number 100 is pretty arbitrary!
if number_files < 100:
export_output.delay(
output_folder_id, mesh_file_id, solution_files)
# Break into chunks a run in parallel
else:
for chunk in [solution_files[i::NUMBER__OF_EXPORT_TASKS] for i in xrange(NUMBER__OF_EXPORT_TASKS)]:
export_output.delay(output_folder_id, mesh_file_id, chunk)
def setup_input(task, *args, **kwargs):
input_folder_id = kwargs['input']['folder']['id']
mesh_file_id = kwargs['input']['meshFile']['id']
kwargs['meshFileId'] = mesh_file_id
number_of_procs = kwargs.get('numberOfSlots')
if not number_of_procs:
number_of_procs = kwargs.get('numberOfNodes')
if not number_of_procs:
size = parse('cluster.config.launch.params.node_instance_count').find(kwargs)
if size:
number_of_procs = size[0].value + 1
else:
raise Exception('Unable to extract number of nodes in cluster')
if not number_of_procs:
raise Exception('Unable to determine number of mpi processes to run.')
number_of_procs = int(number_of_procs)
kwargs['numberOfProcs'] = number_of_procs
client = create_girder_client(
task.taskflow.girder_api_url, task.taskflow.girder_token)
# Get the mesh file metadata to see if we need to import
mesh_file = client.getResource('file/%s' % mesh_file_id)
import_mesh = True
if PYFR_MESH_EXT in mesh_file['exts']:
task.logger.info('Mesh is already in pyfrm format.')
kwargs['meshFilename'] = mesh_file['name']
import_mesh = False
if import_mesh or number_of_procs > 1:
task.logger.info('Downloading input mesh.')
try:
_, input_path = tempfile.mkstemp()
output_dir = tempfile.mkdtemp()
mesh_filename = '%s.pyfrm' % (mesh_file['name'].rsplit('.', 1)[0])
output_path = os.path.join(output_dir, mesh_filename)
kwargs['meshFilename'] = mesh_filename
client.downloadFile(mesh_file_id, input_path)
task.logger.info('Downloading complete.')
if import_mesh:
task.taskflow.logger.info('Importing mesh into PyFr format.')
extn = mesh_file['exts'][0]
task.logger.info('Converting mesh to pyfrm format.')
_import_mesh(task.taskflow.logger, input_path, output_path, extn)
task.logger.info('Conversion complete.')
task.logger.info('Partitioning the mesh.')
if number_of_procs > 1:
_partition_mesh(
task.logger, output_path, output_dir, number_of_procs)
else:
task.logger.info('Skipping partitioning we are running on a single node.')
task.logger.info('Partitioning complete.')
task.logger.info('Uploading converted mesh.')
size = os.path.getsize(output_path)
with open(output_path) as fp:
girder_file = client.uploadFile(
input_folder_id, fp, mesh_filename, size=size,
parentType='folder')
kwargs['meshFileId'] = girder_file['_id']
task.logger.info('Upload input complete.')
task.logger.info('Updating backend configuration.')
finally:
if os.path.exists(input_path):
os.remove(input_path)
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
# If we are running in the cloud determine backend to use
if kwargs['cluster']['type'] == 'ec2':
pyfr_config = {}
# If we have GPUs use cuda
if has_gpus(kwargs['cluster']):
backend = {
'type': 'cuda',
'device-id': 'round-robin'
}
# Update cluster configuation
pyfr_config['cuda'] = True
# Use OpenMP
else:
backend = {
'type': 'openmp',
'cblas': '/usr/lib/openblas-base/libblas.so'
}
# Update cluster configuation
pyfr_config['openmp'] = [{
'name' : 'ec2profile',
'cblas' : "/usr/lib/libblas/libblas.so",
}]
kwargs['backend'] = backend
# Patch cluster in Girder
client.patch('clusters/%s' % kwargs['cluster']['_id'],
data=json.dumps({
'config': {
'pyfr': pyfr_config
}
}))
update_config_file(task, client, *args, **kwargs)
ini_file_id = kwargs['input']['iniFile']['id']
ini_file = client.getResource('file/%s' % ini_file_id)
kwargs['iniFilename'] = ini_file['name']
create_job.delay(*args, **kwargs)
def start(task, input_, cluster, image, run_parameters):
"""
The flow is the following:
- Dry run the container with the -d flag to obtain a description of the input/output formats
- Convert the cjson input geometry to conform to the container's expected format
- Run the container
- Convert the container output format into cjson
- Ingest the output in the database
"""
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
if cluster.get('name') == 'cori':
cluster = _get_cori(client)
if '_id' not in cluster:
raise Exception('Invalid cluster configurations: %s' % cluster)
oc_folder = _get_oc_folder(client)
root_folder = client.createFolder(
oc_folder['_id'],
datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%f"))
# temporary folder to save the container in/out description
description_folder = client.createFolder(root_folder['_id'], 'description')
# save the pull.py script to the job directory
with open(os.path.join(os.path.dirname(__file__), 'utils/pull.py'),
'rb') as f:
# Get the size of the file
size = f.seek(0, 2)
f.seek(0)
name = 'pull.py'
input_parameters_file = client.uploadFile(description_folder['_id'],
f,
name,
size,
parentType='folder')
job = _create_description_job(task, cluster, description_folder, image,
run_parameters)
# Now download pull.py script to the cluster
task.taskflow.logger.info(
'Downloading description input files to cluster.')
download_job_input_folders(cluster,
job,
girder_token=task.taskflow.girder_token,
submit=False)
task.taskflow.logger.info('Downloading complete.')
submit_job(cluster,
job,
girder_token=task.taskflow.girder_token,
monitor=False)
monitor_job.apply_async(
(cluster, job), {
'girder_token': task.taskflow.girder_token,
'monitor_interval': 10
},
link=postprocess_description.s(input_, cluster, image, run_parameters,
root_folder, job, description_folder))
def postprocess_description(task, _, input_, cluster, image, run_parameters,
root_folder, description_job, description_folder):
task.taskflow.logger.info('Processing description job output.')
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
# Refresh state of job
description_job = client.get('jobs/%s' % description_job['_id'])
upload_job_output_to_folder(cluster,
description_job,
girder_token=task.taskflow.girder_token)
description_items = list(client.listItem(description_folder['_id']))
description_file = None
pull_file = None
for item in description_items:
if item['name'] == 'description.json':
files = list(client.listFile(item['_id']))
if len(files) != 1:
raise Exception(
'Expecting a single file under item, found: %s' +
len(files))
description_file = files[0]
elif item['name'] == 'pull.json':
files = list(client.listFile(item['_id']))
if len(files) != 1:
raise Exception(
'Expecting a single file under item, found: %s' +
len(files))
pull_file = files[0]
if pull_file is None:
raise Exception(
'There was an error trying to pull the requested container image')
if description_file is None:
raise Exception(
'The container does not implement correctly the --description flag'
)
with tempfile.TemporaryFile() as tf:
client.downloadFile(pull_file['_id'], tf)
tf.seek(0)
container_pull = json.loads(tf.read().decode())
image = container_pull
with tempfile.TemporaryFile() as tf:
client.downloadFile(description_file['_id'], tf)
tf.seek(0)
container_description = json.loads(tf.read().decode())
# remove temporary description folder
client.delete('folder/%s' % description_folder['_id'])
setup_input.delay(input_, cluster, image, run_parameters, root_folder,
container_description)
def setup_input(task, input_, cluster, image, run_parameters, root_folder,
container_description):
task.taskflow.logger.info('Setting up calculation input.')
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
if cluster.get('name') == 'cori':
cluster = _get_cori(client)
if '_id' not in cluster:
raise Exception('Invalid cluster configurations: %s' % cluster)
calculation_id = parse('calculation._id').find(input_)
if not calculation_id:
raise Exception('Unable to extract calculation id.')
calculation_id = calculation_id[0].value
calculation = client.get('calculations/%s' % calculation_id)
molecule_id = calculation['moleculeId']
input_parameters = calculation.get('input', {}).get('parameters', {})
# Fetch the starting geometry
input_geometry = calculation.get('input', {}).get('geometry', None)
if input_geometry is None:
r = client.get('molecules/%s/cjson' % molecule_id, jsonResp=False)
cjson = r.json()
else:
# TODO: implement the path where a specific input geometry exists
raise NotImplementedError(
'Running a calculation with a specific geometry is not implemented yet.'
)
input_format = container_description['input']['format']
output_format = container_description['output']['format']
# The folder where the input geometry and input parameters are
input_folder = client.createFolder(root_folder['_id'], 'input')
# The folder where the converted output will be at the end of the job
output_folder = client.createFolder(root_folder['_id'], 'output')
# The folder where the raw input/output files of the specific code are stored
scratch_folder = client.createFolder(root_folder['_id'], 'scratch')
# Save the input parameters to file
with tempfile.TemporaryFile() as fp:
fp.write(json.dumps(input_parameters).encode())
# Get the size of the file
size = fp.seek(0, 2)
fp.seek(0)
name = 'input_parameters.json'
input_parameters_file = client.uploadFile(input_folder['_id'],
fp,
name,
size,
parentType='folder')
# Save the input geometry to file
with tempfile.TemporaryFile() as fp:
content = _convert_geometry(cjson, input_format)
fp.write(content.encode())
# Get the size of the file
size = fp.seek(0, 2)
fp.seek(0)
name = 'geometry.%s' % input_format
input_geometry_file = client.uploadFile(input_folder['_id'],
fp,
name,
size,
parentType='folder')
submit_calculation.delay(input_, cluster, image, run_parameters,
root_folder, container_description, input_folder,
output_folder, scratch_folder)
def _create_job(task, cluster, image, run_parameters, container_description,
input_folder, output_folder, scratch_folder):
params = _get_job_parameters(cluster, image, run_parameters)
container = params['container']
image_uri = params['imageUri']
repository = params['repository']
digest = params['digest']
host_dir = params['hostDir']
guest_dir = params['guestDir']
job_dir = params['jobDir']
task.taskflow.logger.info('Create %s job' % repository)
input_format = container_description['input']['format']
output_format = container_description['output']['format']
input_dir = os.path.join(guest_dir, job_dir, 'input')
output_dir = os.path.join(guest_dir, job_dir, 'output')
scratch_dir = os.path.join(guest_dir, job_dir, 'scratch')
geometry_filename = os.path.join(input_dir, 'geometry.%s' % input_format)
parameters_filename = os.path.join(input_dir, 'input_parameters.json')
output_filename = os.path.join(output_dir, 'output.%s' % output_format)
output = [{'folderId': output_folder['_id'], 'path': './output'}]
keep_scratch = run_parameters.get('keepScratch', False)
if keep_scratch:
output.append({'folderId': scratch_folder['_id'], 'path': './scratch'})
if container == 'docker':
mount_option = '-v %s:%s' % (host_dir, guest_dir)
else:
mount_option = ''
body = {
# ensure there are no special characters in the submission script name
'name':
'run_%s' % re.sub('[^a-zA-Z0-9]', '_', repository),
'commands': [
'mkdir output', 'mkdir scratch',
'%s run %s %s -g %s -p %s -o %s -s %s' %
(container, mount_option, image_uri, geometry_filename,
parameters_filename, output_filename, scratch_dir)
],
'input': [{
'folderId': input_folder['_id'],
'path': './input'
}],
'output':
output,
'uploadOutput':
False,
'params': {
'taskFlowId': task.taskflow.id
}
}
client = create_girder_client(task.taskflow.girder_api_url,
task.taskflow.girder_token)
job = client.post('jobs', data=json.dumps(body))
task.taskflow.set_metadata('jobs', [job])
return job