def remote_side_bash_executor(func, *args, **kwargs):
"""Executes the supplied function with *args and **kwargs to get a
command-line to run, and then run that command-line using bash.
"""
import os
import subprocess
import parsl.app.errors as pe
from parsl.utils import get_std_fname_mode
if hasattr(func, '__name__'):
func_name = func.__name__
else:
logger.warning(
'No name for the function. Potentially a result of parsl#2233')
func_name = 'bash_app'
executable = None
# Try to run the func to compose the commandline
try:
# Execute the func to get the commandline
executable = func(*args, **kwargs)
if not isinstance(executable, str):
raise ValueError(
f"Expected a str for bash_app commandline, got {type(executable)}"
)
except AttributeError as e:
if executable is not None:
raise pe.AppBadFormatting(
"App formatting failed for app '{}' with AttributeError: {}".
format(func_name, e))
else:
raise pe.BashAppNoReturn(
"Bash app '{}' did not return a value, or returned None - with this exception: {}"
.format(func_name, e))
except IndexError as e:
raise pe.AppBadFormatting(
"App formatting failed for app '{}' with IndexError: {}".format(
func_name, e))
except Exception as e:
raise e
# Updating stdout, stderr if values passed at call time.
def open_std_fd(fdname):
# fdname is 'stdout' or 'stderr'
stdfspec = kwargs.get(fdname) # spec is str name or tuple (name, mode)
if stdfspec is None:
return None
fname, mode = get_std_fname_mode(fdname, stdfspec)
try:
if os.path.dirname(fname):
os.makedirs(os.path.dirname(fname), exist_ok=True)
fd = open(fname, mode)
except Exception as e:
raise pe.BadStdStreamFile(fname, e)
return fd
std_out = open_std_fd('stdout')
std_err = open_std_fd('stderr')
timeout = kwargs.get('walltime')
if std_err is not None:
print('--> executable follows <--\n{}\n--> end executable <--'.format(
executable),
file=std_err,
flush=True)
returncode = None
try:
proc = subprocess.Popen(executable,
stdout=std_out,
stderr=std_err,
shell=True,
executable='/bin/bash',
close_fds=False)
proc.wait(timeout=timeout)
returncode = proc.returncode
except subprocess.TimeoutExpired:
raise pe.AppTimeout("[{}] App exceeded walltime: {} seconds".format(
func_name, timeout))
except Exception as e:
raise pe.AppException(
"[{}] App caught exception with returncode: {}".format(
func_name, returncode), e)
if returncode != 0:
raise pe.BashExitFailure(func_name, proc.returncode)
# TODO : Add support for globs here
missing = []
for outputfile in kwargs.get('outputs', []):
fpath = outputfile.filepath
if not os.path.exists(fpath):
missing.extend([outputfile])
if missing:
raise pe.MissingOutputs("[{}] Missing outputs".format(func_name),
missing)
return returncode
def remote_side_sandbox_executor(func, *args, **kwargs):
"""Executes the supplied function with *args and **kwargs to get a
command-line to run, and then run that command-line using bash.
"""
import os
import time
import subprocess
import logging
import parsl.app.errors as pe
from parsl import set_file_logger
from parsl.utils import get_std_fname_mode
sandbox = Sandbox("scratch")
logbase = "/tmp"
format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s] %(message)s"
# make this name unique per invocation so that each invocation can
# log to its own file. It would be better to include the task_id here
# but that is awkward to wire through at the moment as apps do not
# have access to that execution context.
t = time.time()
logname = __name__ + "." + str(t)
logger = logging.getLogger(logname)
set_file_logger(filename='{0}/bashexec.{1}.log'.format(logbase, t),
name=logname,
level=logging.DEBUG,
format_string=format_string)
func_name = func.__name__
executable = None
# Try to run the func to compose the commandline
try:
# Execute the func to get the commandline
executable = func(*args, **kwargs)
except AttributeError as e:
if executable is not None:
raise pe.AppBadFormatting(
"App formatting failed for app '{}' with AttributeError: {}".
format(func_name, e))
else:
raise pe.BashAppNoReturn(
"Bash app '{}' did not return a value, or returned None - with this exception: {}"
.format(func_name, e))
except IndexError as e:
raise pe.AppBadFormatting(
"App formatting failed for app '{}' with IndexError: {}".format(
func_name, e))
except Exception as e:
logger.error(
"Caught exception during formatting of app '{}': {}".format(
func_name, e))
raise e
logger.debug("Executable: %s", executable)
# Updating stdout, stderr if values passed at call time.
def open_std_fd(fdname):
# fdname is 'stdout' or 'stderr'
stdfspec = kwargs.get(fdname) # spec is str name or tuple (name, mode)
if stdfspec is None:
return None
fname, mode = get_std_fname_mode(fdname, stdfspec)
try:
if os.path.dirname(fname):
os.makedirs(os.path.dirname(fname), exist_ok=True)
fd = open(fname, mode)
except Exception as e:
raise pe.BadStdStreamFile(fname, e)
return fd
std_out = open_std_fd('stdout')
std_err = open_std_fd('stderr')
timeout = kwargs.get('walltime')
project = kwargs.get('project', "")
unique_id = kwargs.get('unique_id', "HUMAN")
sandbox.create_working_dir(unique_id, func_name)
workflow_schema = "workflow://" + project + "/" + unique_id + "/"
if std_err is not None:
print('--> executable follows <--\n{}\n--> end executable <--'.format(
executable),
file=std_err,
flush=True)
return_value = None
try:
cwd = None
working_directory = "scratch" + os.path.sep + unique_id
os.makedirs(working_directory)
cwd = os.getcwd()
os.chdir(working_directory)
logger.debug("workflow://schema: %s", workflow_schema)
# Resolve workflow:// inputs
for input in kwargs.get('inputs', []):
if "workflow://" in input:
print(input)
proc = subprocess.Popen(executable,
stdout=std_out,
stderr=std_err,
shell=True,
executable='/bin/bash')
proc.wait(timeout=timeout)
return_value = {
'unique_id': unique_id,
'working_directory': working_directory,
'workflow_schema': workflow_schema,
'return_code': proc.returncode
}
if cwd is not None:
os.chdir(cwd)
except subprocess.TimeoutExpired:
raise pe.AppTimeout("[{}] App exceeded walltime: {}".format(
func_name, timeout))
except Exception as e:
raise pe.AppException(
"[{}] App caught exception with return value: {}".format(
func_name, json.dumps(return_value)), e)
if proc.returncode != 0:
raise pe.BashExitFailure(func_name, proc.returncode)
# TODO : Add support for globs here
missing = []
for outputfile in kwargs.get('outputs', []):
fpath = outputfile.filepath
if not os.path.exists(fpath):
missing.extend([outputfile])
if missing:
raise pe.MissingOutputs("[{}] Missing outputs".format(func_name),
missing)
return return_value
def remote_side_bash_executor(func, *args, **kwargs):
"""Executes the supplied function with *args and **kwargs to get a
command-line to run, and then run that command-line using bash.
"""
import os
import time
import subprocess
import logging
import parsl.app.errors as pe
from parsl import set_file_logger
from parsl.utils import get_std_fname_mode
logbase = "/tmp"
format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s] %(message)s"
# make this name unique per invocation so that each invocation can
# log to its own file. It would be better to include the task_id here
# but that is awkward to wire through at the moment as apps do not
# have access to that execution context.
t = time.time()
logname = __name__ + "." + str(t)
logger = logging.getLogger(logname)
set_file_logger(filename='{0}/bashexec.{1}.log'.format(logbase, t),
name=logname,
level=logging.DEBUG,
format_string=format_string)
func_name = func.__name__
executable = None
# Try to run the func to compose the commandline
try:
# Execute the func to get the commandline
executable = func(*args, **kwargs)
if not isinstance(executable, str):
raise ValueError(
f"Expected a str for bash_app commandline, got {type(executable)}"
)
except AttributeError as e:
if executable is not None:
raise pe.AppBadFormatting(
"App formatting failed for app '{}' with AttributeError: {}".
format(func_name, e))
else:
raise pe.BashAppNoReturn(
"Bash app '{}' did not return a value, or returned None - with this exception: {}"
.format(func_name, e))
except IndexError as e:
raise pe.AppBadFormatting(
"App formatting failed for app '{}' with IndexError: {}".format(
func_name, e))
except Exception as e:
logger.error(
"Caught exception during formatting of app '{}': {}".format(
func_name, e))
raise e
logger.debug("Executable: %s", executable)
# Updating stdout, stderr if values passed at call time.
def open_std_fd(fdname):
# fdname is 'stdout' or 'stderr'
stdfspec = kwargs.get(fdname) # spec is str name or tuple (name, mode)
if stdfspec is None:
return None
fname, mode = get_std_fname_mode(fdname, stdfspec)
try:
if os.path.dirname(fname):
os.makedirs(os.path.dirname(fname), exist_ok=True)
fd = open(fname, mode)
except Exception as e:
raise pe.BadStdStreamFile(fname, e)
return fd
std_out = open_std_fd('stdout')
std_err = open_std_fd('stderr')
timeout = kwargs.get('walltime')
if std_err is not None:
print('--> executable follows <--\n{}\n--> end executable <--'.format(
executable),
file=std_err,
flush=True)
returncode = None
try:
proc = subprocess.Popen(executable,
stdout=std_out,
stderr=std_err,
shell=True,
executable='/bin/bash')
proc.wait(timeout=timeout)
returncode = proc.returncode
except subprocess.TimeoutExpired:
raise pe.AppTimeout("[{}] App exceeded walltime: {}".format(
func_name, timeout))
except Exception as e:
raise pe.AppException(
"[{}] App caught exception with returncode: {}".format(
func_name, returncode), e)
if returncode != 0:
raise pe.BashExitFailure(func_name, proc.returncode)
# TODO : Add support for globs here
missing = []
for outputfile in kwargs.get('outputs', []):
fpath = outputfile.filepath
if not os.path.exists(fpath):
missing.extend([outputfile])
if missing:
raise pe.MissingOutputs("[{}] Missing outputs".format(func_name),
missing)
return returncode
def sandbox_runner(func, *args, **kwargs):
"""Executes the supplied function with *args and **kwargs to get a
command-line to run, and then run that command-line using bash.
"""
import os
import time
import subprocess
import logging
import parsl.app.errors as pe
from parsl import set_file_logger
from parsl.utils import get_std_fname_mode
from parsl.data_provider.files import File
import json
# create a sandbox passing the name of the scratch directory
sandbox = Sandbox()
logbase = "/tmp"
format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s] %(message)s"
# make this name unique per invocation so that each invocation can
# log to its own file. It would be better to include the task_id here
# but that is awkward to wire through at the moment as apps do not
# have access to that execution context.
t = time.time()
logname = __name__ + "." + str(t)
logger = logging.getLogger(logname)
set_file_logger(filename='{0}/bashexec.{1}.log'.format(logbase, t), name=logname, level=logging.DEBUG,
format_string=format_string)
func_name = func.__name__
executable = None
# the workflow_name
sandbox.workflow_name = kwargs.get('project', "")
# app name
sandbox.app_name = kwargs.get("workflow_app_name", "")
# create a working dir with the sandbox
sandbox.createWorkingDirectory()
# workflow schema as workflow:///funcNameUUID
workflow_schema = "workflow://" + sandbox.workflow_name + "/" + sandbox.app_name
# tasks dep of the current task
if 'tasks' in kwargs:
sandbox.tasks_dep = kwargs.get('tasks', "")
logger.debug(sandbox.tasks_dep)
# Try to run the func to compose the commandline
try:
# Execute the func to get the commandline
executable = func(*args, **kwargs)
executable = sandbox.define_command(executable)
logger.debug(executable)
except AttributeError as e:
if executable is not None:
raise pe.AppBadFormatting("App formatting failed for app '{}' with AttributeError: {}".format(func_name, e))
else:
raise pe.BashAppNoReturn(
"Bash app '{}' did not return a value, or returned None - with this exception: {}".format(func_name, e))
except IndexError as e:
raise pe.AppBadFormatting("App formatting failed for app '{}' with IndexError: {}".format(func_name, e))
except Exception as e:
raise e
# Updating stdout, stderr if values passed at call time.
def open_std_fd(fdname):
# fdname is 'stdout' or 'stderr'
stdfspec = kwargs.get(fdname) # spec is str name or tuple (name, mode)
if stdfspec is None:
return None
fname, mode = get_std_fname_mode(fdname, stdfspec)
try:
if os.path.dirname(fname):
os.makedirs(os.path.dirname(fname), exist_ok=True)
fd = open(fname, mode)
except Exception as e:
raise pe.BadStdStreamFile(fname, e)
return fd
std_out = open_std_fd('stdout')
std_err = open_std_fd('stderr')
timeout = kwargs.get('walltime')
if std_err is not None:
print('--> executable follows <--\n{}\n--> end executable <--'.format(executable), file=std_err, flush=True)
return_value = None
try:
logger.debug("workflow://schema: %s", workflow_schema)
proc = subprocess.Popen(executable, stdout=std_out, stderr=std_err, shell=True, executable='/bin/bash')
proc.wait(timeout=timeout)
return_value = {
'return_code': proc.returncode,
'working_directory': sandbox.working_directory,
}
except subprocess.TimeoutExpired:
raise pe.AppTimeout("[{}] App exceeded walltime: {}".format(func_name, timeout))
except Exception as e:
raise pe.AppException("[{}] App caught exception with return value: {}"
.format(func_name, json.dumps(return_value)), e)
if proc.returncode != 0:
raise pe.BashExitFailure(func_name, proc.returncode)
# TODO : Add support for globs here
missing = []
for outputfile in kwargs.get('outputs', []):
fpath = outputfile.filepath
if not os.path.exists(fpath):
missing.extend([outputfile])
if missing:
raise pe.MissingOutputs("[{}] Missing outputs".format(func_name), missing)
return return_value