def _command(self, environment, code_package_url, step_name, step_cmds,
task_spec):
mflog_expr = export_mflog_env_vars(datastore_type="s3",
stdout_path=STDOUT_PATH,
stderr_path=STDERR_PATH,
**task_spec)
init_cmds = environment.get_package_commands(code_package_url)
init_expr = " && ".join(init_cmds)
step_expr = bash_capture_logs(
" && ".join(environment.bootstrap_commands(step_name) + step_cmds))
# construct an entry point that
# 1) initializes the mflog environment (mflog_expr)
# 2) bootstraps a metaflow environment (init_expr)
# 3) executes a task (step_expr)
# the `true` command is to make sure that the generated command
# plays well with docker containers which have entrypoint set as
# eval $@
cmd_str = "true && mkdir -p /logs && %s && %s && %s; " % (
mflog_expr,
init_expr,
step_expr,
)
# after the task has finished, we save its exit code (fail/success)
# and persist the final logs. The whole entrypoint should exit
# with the exit code (c) of the task.
#
# Note that if step_expr OOMs, this tail expression is never executed.
# We lose the last logs in this scenario (although they are visible
# still through AWS CloudWatch console).
cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS
return shlex.split('bash -c "%s"' % cmd_str)
def _command(self, environment, code_package_url, step_name, step_cmds,
task_spec):
mflog_expr = export_mflog_env_vars(datastore_type='s3',
stdout_path=STDOUT_PATH,
stderr_path=STDERR_PATH,
**task_spec)
init_cmds = environment.get_package_commands(code_package_url)
init_cmds.extend(environment.bootstrap_commands(step_name))
init_expr = ' && '.join(init_cmds)
step_expr = bash_capture_logs(' && '.join(step_cmds))
# construct an entry point that
# 1) initializes the mflog environment (mflog_expr)
# 2) bootstraps a metaflow environment (init_expr)
# 3) executes a task (step_expr)
cmd_str = 'mkdir -p /logs && %s && %s && %s; ' % \
(mflog_expr, init_expr, step_expr)
# after the task has finished, we save its exit code (fail/success)
# and persist the final logs. The whole entrypoint should exit
# with the exit code (c) of the task.
#
# Note that if step_expr OOMs, this tail expression is never executed.
# We lose the last logs in this scenario (although they are visible
# still through AWS CloudWatch console).
cmd_str += 'c=$?; %s; exit $c' % BASH_SAVE_LOGS
return shlex.split('bash -c \"%s\"' % cmd_str)
def _command(
self,
flow_name,
run_id,
step_name,
task_id,
attempt,
code_package_url,
step_cmds,
):
mflog_expr = export_mflog_env_vars(
flow_name=flow_name,
run_id=run_id,
step_name=step_name,
task_id=task_id,
retry_count=attempt,
datastore_type=self._datastore.TYPE,
stdout_path=STDOUT_PATH,
stderr_path=STDERR_PATH,
)
init_cmds = self._environment.get_package_commands(code_package_url)
init_expr = " && ".join(init_cmds)
step_expr = bash_capture_logs(" && ".join(
self._environment.bootstrap_commands(step_name) + step_cmds))
# Construct an entry point that
# 1) initializes the mflog environment (mflog_expr)
# 2) bootstraps a metaflow environment (init_expr)
# 3) executes a task (step_expr)
# The `true` command is to make sure that the generated command
# plays well with docker containers which have entrypoint set as
# eval $@
cmd_str = "true && mkdir -p %s && %s && %s && %s; " % (
LOGS_DIR,
mflog_expr,
init_expr,
step_expr,
)
# After the task has finished, we save its exit code (fail/success)
# and persist the final logs. The whole entrypoint should exit
# with the exit code (c) of the task.
#
# Note that if step_expr OOMs, this tail expression is never executed.
# We lose the last logs in this scenario.
#
# TODO: Capture hard exit logs in Kubernetes.
cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS
# For supporting sandboxes, ensure that a custom script is executed before
# anything else is executed. The script is passed in as an env var.
cmd_str = (
'${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s'
% cmd_str)
return shlex.split('bash -c "%s"' % cmd_str)
def _command(
self,
code_package_url,
step_cmds,
):
mflog_expr = export_mflog_env_vars(
flow_name=self._flow_name,
run_id=self._run_id,
step_name=self._step_name,
task_id=self._task_id,
retry_count=self._attempt,
datastore_type=self._datastore.TYPE,
stdout_path=STDOUT_PATH,
stderr_path=STDERR_PATH,
)
init_cmds = self._environment.get_package_commands(code_package_url)
init_expr = " && ".join(init_cmds)
step_expr = " && ".join([
capture_output_to_mflog(a)
for a in (self._environment.bootstrap_commands(self._step_name) +
step_cmds)
])
# Construct an entry point that
# 1) initializes the mflog environment (mflog_expr)
# 2) bootstraps a metaflow environment (init_expr)
# 3) executes a task (step_expr)
# The `true` command is to make sure that the generated command
# plays well with docker containers which have entrypoint set as
# eval $@
cmd_str = "true && mkdir -p %s && %s && %s && %s; " % (
LOGS_DIR,
mflog_expr,
init_expr,
step_expr,
)
# After the task has finished, we save its exit code (fail/success)
# and persist the final logs. The whole entrypoint should exit
# with the exit code (c) of the task.
#
# Note that if step_expr OOMs, this tail expression is never executed.
# We lose the last logs in this scenario.
#
# TODO: Find a way to capture hard exit logs in Kubernetes.
cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS
return shlex.split('bash -c "%s"' % cmd_str)
def _commands(self, node, retry_count, user_code_retries):
mflog_expr = export_mflog_env_vars(datastore_type='s3',
stdout_path='/tmp/mflog_stdout',
stderr_path='/tmp/mflog_stderr',
flow_name=self.flow.name,
run_id='{{workflow.name}}',
step_name=node.name,
task_id='{{pod.name}}',
retry_count=retry_count)
init_cmds = []
if self.code_package_url:
init_cmds.extend(
self.environment.get_package_commands(self.code_package_url))
init_cmds.extend(self.environment.bootstrap_commands(node.name))
init_expr = ' && '.join(init_cmds)
step_expr = " && ".join([
capture_output_to_mflog(a) for a in (
self._step_commands(node, retry_count, user_code_retries))
])
cmd = ['true', mflog_expr, init_expr, step_expr]
cmd_str = '%s; c=$?; %s; exit $c' % (' && '.join(
c for c in cmd if c), BASH_SAVE_LOGS)
return shlex.split('bash -c \"%s\"' % cmd_str)