def step_init(self, flow, graph, step, decos, environment, flow_datastore,
logger):
# Executing Kubernetes jobs requires a non-local datastore at the
# moment.
# TODO: To support MiniKube we need to enable local datastore execution.
if flow_datastore.TYPE != "s3":
raise KubernetesException(
"The *@kubernetes* decorator requires --datastore=s3 "
"at the moment.")
# Set internal state.
self.logger = logger
self.environment = environment
self.step = step
self.flow_datastore = flow_datastore
for deco in decos:
if isinstance(deco, ResourcesDecorator):
for k, v in deco.attributes.items():
# We use the larger of @resources and @k8s attributes
# TODO: Fix https://github.com/Netflix/metaflow/issues/467
my_val = self.attributes.get(k)
if not (my_val is None and v is None):
self.attributes[k] = str(
max(int(my_val or 0), int(v or 0)))
# Set run time limit for the Kubernetes job.
self.run_time_limit = get_run_time_limit_for_task(decos)
if self.run_time_limit < 60:
raise KubernetesException(
"The timeout for step *{step}* should be "
"at least 60 seconds for execution on "
"Kubernetes.".format(step=step))
def step_init(self, flow, graph, step, decos, environment, flow_datastore,
logger):
if flow_datastore.TYPE != "s3":
raise BatchException(
"The *@batch* decorator requires --datastore=s3.")
# Set internal state.
self.logger = logger
self.environment = environment
self.step = step
self.flow_datastore = flow_datastore
for deco in decos:
if isinstance(deco, ResourcesDecorator):
for k, v in deco.attributes.items():
# We use the larger of @resources and @batch attributes
# TODO: Fix https://github.com/Netflix/metaflow/issues/467
my_val = self.attributes.get(k)
if not (my_val is None and v is None):
self.attributes[k] = str(
max(int(my_val or 0), int(v or 0)))
# Set run time limit for the AWS Batch job.
self.run_time_limit = get_run_time_limit_for_task(decos)
if self.run_time_limit < 60:
raise BatchException(
"The timeout for step *{step}* should be at "
"least 60 seconds for execution on AWS Batch.".format(
step=step))
def step_init(self, flow, graph, step, decos, environment, datastore, logger):
if datastore.TYPE != 's3':
raise BatchException('The *@batch* decorator requires --datastore=s3.')
self.logger = logger
self.environment = environment
self.step = step
for deco in decos:
if isinstance(deco, ResourcesDecorator):
for k, v in deco.attributes.items():
# we use the larger of @resources and @batch attributes
my_val = self.attributes.get(k)
if not (my_val is None and v is None):
self.attributes[k] = str(max(int(my_val or 0), int(v or 0)))
self.run_time_limit = get_run_time_limit_for_task(decos)
if self.run_time_limit < 60:
raise BatchException('The timeout for step *{step}* should be at '
'least 60 seconds for execution on AWS Batch'.format(step=step))
def step_init(self, flow, graph, step, decos, environment, flow_datastore,
logger):
if flow_datastore.TYPE != "s3":
raise BatchException(
"The *@batch* decorator requires --datastore=s3.")
# Set internal state.
self.logger = logger
self.environment = environment
self.step = step
self.flow_datastore = flow_datastore
self.attributes.update(
compute_resource_attributes(decos, self, self.resource_defaults))
# Set run time limit for the AWS Batch job.
self.run_time_limit = get_run_time_limit_for_task(decos)
if self.run_time_limit < 60:
raise BatchException(
"The timeout for step *{step}* should be at "
"least 60 seconds for execution on AWS Batch.".format(
step=step))
def container_template(self, node):
"""
Returns an argo container template spec. to execute a step
"""
attr = parse_step_decorator(node, ArgoStepDecorator)
env_decorator = parse_step_decorator(node, EnvironmentDecorator)
retry_decorator = parse_step_decorator(node, RetryDecorator)
catch_decorator = parse_step_decorator(node, CatchDecorator)
res_decorator = parse_step_decorator(node, ResourcesDecorator)
k8s_decorator = parse_step_decorator(node, KubernetesDecorator)
resources = merge_resources(
res_decorator, {
k: v
for k, v in k8s_decorator.items()
if k in ResourcesDecorator.defaults
})
image = attr.get('image') or k8s_decorator.get(
'image') or self._default_image()
env, env_from = self._prepare_environment(attr, env_decorator)
res = self._resources(resources)
volume_mounts = attr.get('volumeMounts', [])
volume_mounts.append(self._shared_memory(resources))
user_code_retries = retry_decorator.get('times', 0)
total_retries = user_code_retries + 1 if catch_decorator else user_code_retries
retry_count = '{{retries}}' if total_retries else '0'
cmd = self._commands(node, retry_count, user_code_retries)
metadata = {
'labels': {
**attr.get('labels', {}),
**self.attributes['labels'],
'metaflow/step_name':
sanitize_label_value(dns_name(node.name)),
'app.kubernetes.io/name':
'metaflow-task',
'app.kubernetes.io/part-of':
'metaflow',
'app.kubernetes.io/created-by':
get_username(),
},
'annotations': {
**attr.get('annotations', {}),
**self.attributes['annotations'],
# should be a label but cannot sanitize argo variables
'metaflow/attempt':
retry_count,
},
}
metadata['labels'].update(self.system_tags)
template = {
'name': dns_name(node.name),
'metadata': metadata,
'activeDeadlineSeconds':
get_run_time_limit_for_task(node.decorators),
'inputs': {
'parameters': [{
'name': 'input-paths'
}],
'artifacts': attr.get('input_artifacts'),
},
'outputs': {
'parameters': [{
'name': 'task-id',
'value': '{{pod.name}}'
}],
'artifacts': attr.get('output_artifacts')
},
'nodeSelector': attr.get('nodeSelector'),
'container': {
'image': image,
'volumeMounts': volume_mounts,
'command': [cmd[0]],
'args': cmd[1:],
'env': env,
'envFrom': env_from,
'resources': {
'requests': res,
'limits': res
}
},
}
if total_retries:
template['retryStrategy'] = {
'retryPolicy': 'Always',
# fallback_step for @catch is only executed if retry_count > user_code_retries
'limit': str(total_retries),
'backoff': {
'duration':
'%sm' % str(retry_decorator['minutes_between_retries']
if user_code_retries else 0),
}
}
if self._is_foreach_first_child(node):
template['inputs']['parameters'].append({'name': 'split-index'})
if node.type == 'foreach':
template['outputs']['parameters'].append({
'name': 'num-splits',
'valueFrom': {
'path': ArgoInternalStepDecorator.splits_file_path
}
})
return template
def step_init(self, flow, graph, step, decos, environment, flow_datastore,
logger):
# Executing Kubernetes jobs requires a non-local datastore.
if flow_datastore.TYPE != "s3":
raise KubernetesException(
"The *@kubernetes* decorator requires --datastore=s3 at the moment."
)
# Set internal state.
self.logger = logger
self.environment = environment
self.step = step
self.flow_datastore = flow_datastore
if any([deco.name == "batch" for deco in decos]):
raise MetaflowException(
"Step *{step}* is marked for execution both on AWS Batch and "
"Kubernetes. Please use one or the other.".format(step=step))
for deco in decos:
if getattr(deco, "IS_PARALLEL", False):
raise KubernetesException(
"@kubernetes does not support parallel execution currently."
)
# Set run time limit for the Kubernetes job.
self.run_time_limit = get_run_time_limit_for_task(decos)
if self.run_time_limit < 60:
raise KubernetesException(
"The timeout for step *{step}* should be at least 60 seconds for "
"execution on Kubernetes.".format(step=step))
for deco in decos:
if isinstance(deco, ResourcesDecorator):
for k, v in deco.attributes.items():
# TODO: Special case GPUs when they are introduced in @resources.
if k in self.attributes:
if self.defaults[k] is None:
# skip if expected value isn't an int/float
continue
# We use the larger of @resources and @batch attributes
# TODO: Fix https://github.com/Netflix/metaflow/issues/467
my_val = self.attributes.get(k)
if not (my_val is None and v is None):
self.attributes[k] = str(
max(float(my_val or 0), float(v or 0)))
# Check GPU vendor.
if self.attributes["gpu_vendor"].lower() not in ("amd", "nvidia"):
raise KubernetesException(
"GPU vendor *{}* for step *{step}* is not currently supported."
.format(self.attributes["gpu_vendor"], step=step))
# CPU, Disk, and Memory values should be greater than 0.
for attr in ["cpu", "disk", "memory"]:
if not (isinstance(self.attributes[attr],
(int, unicode, basestring, float))
and float(self.attributes[attr]) > 0):
raise KubernetesException(
"Invalid {} value *{}* for step *{step}*; it should be greater than 0"
.format(attr, self.attributes[attr], step=step))
if self.attributes["gpu"] is not None and not (
isinstance(self.attributes["gpu"], (int, unicode, basestring))
and float(self.attributes["gpu"]).is_integer()):
raise KubernetesException(
"Invalid GPU value *{}* for step *{step}*; it should be an integer"
.format(self.attributes["gpu"], step=step))