def safe_value_to_str(value, max_len):
if value is None or isinstance(value, six.string_types):
return safe_short_string(str(value), max_len)
try:
# local import to prevent imports recursion
from targets.values import get_value_type_of_obj
value_type = get_value_type_of_obj(value)
if value_type is not None:
return value_type.to_preview(value, max_len)
except Exception:
logger.warning("failed dumping metric, falling back to str", exc_info=True)
# This object has to hold only serializable values
return safe_short_string(str(value), max_len)
def safe_string(value, max_value_len=1000):
try:
if not isinstance(value, six.string_types):
value = str(value)
return safe_short_string(value=value, max_value_len=max_value_len)
except Exception as ex:
return "Failed to convert value to string:%s" % ex
def get_safe_short_text(text, max_size):
if max_size == -1: # use -1 for unlimited
return text
if max_size == 0: # use 0 to disable
return None
is_tail_preview = (
max_size > 0
) # pass negative to get log's 'head' instead of 'tail'
return safe_short_string(text, abs(max_size), tail=is_tail_preview)
def build_task_run_info(task_run):
# type: (TaskRun) -> TaskRunInfo
t = task_run.task
tm = task_run.task.task_meta
task_dag = t.ctrl.task_dag
log_local, log_remote = task_run._get_log_files()
task_params_values = dict(t._params.get_params_serialized())
task_definition = t.task_definition
task_run_params = [
TaskRunParamInfo(
parameter_name=tdp.name,
value_origin=t._params.get_param_value_origin(tdp.name),
value=safe_short_string(task_params_values[tdp.name],
max_value_len=5000),
) for tdp in task_definition.task_params.values()
]
return TaskRunInfo(
run_uid=task_run.run.run_uid,
task_definition_uid=task_run.task.task_definition.task_definition_uid,
task_run_uid=task_run.task_run_uid, # this is not the TaskRun uid
task_run_attempt_uid=task_run.
task_run_attempt_uid, # this is not the TaskRun uid
task_id=t.task_id,
task_af_id=task_run.task_af_id,
name=t.task_name,
task_signature=tm.task_signature,
task_signature_source=tm.task_signature_source,
output_signature=tm.task_outputs_signature,
command_line=tm.task_command_line,
env=t.task_env.name,
functional_call=tm.task_functional_call,
has_downstreams=bool(task_dag.downstream),
has_upstreams=bool(task_dag.upstream),
state=TaskRunState.SCHEDULED
if not task_run.is_reused else TaskRunState.SUCCESS,
is_reused=task_run.is_reused,
is_skipped=task_run.is_skipped,
is_dynamic=task_run.is_dynamic,
is_system=task_run.is_system,
version=t.task_version,
target_date=t.task_target_date,
log_local=log_local,
log_remote=log_remote,
task_run_params=task_run_params,
execution_date=task_run.run.execution_date,
is_root=task_run.is_root,
)
def build_task_run_info(task_run):
# type: (TaskRun) -> TaskRunInfo
t = task_run.task
task_dag = t.ctrl.task_dag
log_local, log_remote = task_run._get_log_files()
task_run_params = []
for param_meta in t.task_params.get_param_values():
if isinstance(param_meta.parameter, FuncResultParameter):
continue
if param_meta:
value_source, value = param_meta.source, param_meta.value
param_meta.parameter = (
param_meta.parameter.update_value_meta_conf_from_runtime_value(
value, t.settings.tracking
)
)
else:
value_source, value = "", ""
if (
param_meta.parameter.hidden
or not param_meta.parameter.value_meta_conf.log_preview
):
value = ParameterValue.MASKED_VALUE_PREVIEW
else:
value = safe_short_string(
param_meta.parameter.signature(value), max_value_len=5000
)
task_run_params.append(
TaskRunParamInfo(
parameter_name=param_meta.name,
value_origin=safe_short_string(str(value_source), max_value_len=5000),
value=value,
)
)
return TaskRunInfo(
run_uid=task_run.run.run_uid,
task_definition_uid=task_run.task.task_definition.task_definition_uid,
task_run_uid=task_run.task_run_uid, # this is not the TaskRun uid
task_run_attempt_uid=task_run.task_run_attempt_uid, # this is not the TaskRun uid
task_id=t.task_id,
task_af_id=task_run.task_af_id,
name=t.task_name,
task_signature=t.task_signature_obj.signature,
task_signature_source=t.task_signature_obj.signature_source,
output_signature=t.task_outputs_signature_obj.signature,
command_line=t.ctrl.task_repr.task_command_line,
env=t.task_env.name,
functional_call=t.ctrl.task_repr.task_functional_call,
has_downstreams=bool(task_dag.downstream),
has_upstreams=bool(task_dag.upstream),
state=TaskRunState.SCHEDULED
if not task_run.is_reused
else TaskRunState.SUCCESS,
is_reused=task_run.is_reused,
is_skipped=task_run.is_skipped,
is_dynamic=task_run.is_dynamic,
is_system=task_run.is_system,
version=t.task_version,
target_date=t.task_target_date,
log_local=log_local,
log_remote=log_remote,
task_run_params=task_run_params,
execution_date=task_run.run.execution_date,
is_root=task_run.is_root,
)
def _extract_log_preivew(self, log_body=None, max_size=1000):
is_tail_preview = (
max_size > 0
) # pass negative to get log's 'head' instead of 'tail'
return safe_short_string(log_body, abs(max_size), tail=is_tail_preview)
def build_task_run_info(task_run):
# type: (TaskRun) -> TaskRunInfo
t = task_run.task
tm = task_run.task.task_meta
task_dag = t.ctrl.task_dag
log_local, log_remote = task_run._get_log_files()
task_params_values = dict(t._params.get_params_serialized())
td = t.task_definition
task_run_params = []
for key, tdp in sorted(td.task_params.items()):
if isinstance(tdp, FuncResultParameter):
continue
param_meta = t._params.get_param_meta(tdp.name)
if param_meta:
value_source, value = param_meta.source, param_meta.value
else:
value_source, value = "", ""
task_run_params.append(
TaskRunParamInfo(
parameter_name=tdp.name,
value_origin=safe_short_string(str(value_source),
max_value_len=5000),
value=safe_short_string(
str(task_params_values.get(tdp.name, value))
if not tdp.hidden else "***",
max_value_len=5000,
),
))
return TaskRunInfo(
run_uid=task_run.run.run_uid,
task_definition_uid=task_run.task.task_definition.task_definition_uid,
task_run_uid=task_run.task_run_uid, # this is not the TaskRun uid
task_run_attempt_uid=task_run.
task_run_attempt_uid, # this is not the TaskRun uid
task_id=t.task_id,
task_af_id=task_run.task_af_id,
name=t.task_name,
task_signature=tm.task_signature,
task_signature_source=tm.task_signature_source,
output_signature=tm.task_outputs_signature,
command_line=t.ctrl.task_repr.task_command_line,
env=t.task_env.name,
functional_call=t.ctrl.task_repr.task_functional_call,
has_downstreams=bool(task_dag.downstream),
has_upstreams=bool(task_dag.upstream),
state=TaskRunState.SCHEDULED
if not task_run.is_reused else TaskRunState.SUCCESS,
is_reused=task_run.is_reused,
is_skipped=task_run.is_skipped,
is_dynamic=task_run.is_dynamic,
is_system=task_run.is_system,
version=t.task_version,
target_date=t.task_target_date,
log_local=log_local,
log_remote=log_remote,
task_run_params=task_run_params,
execution_date=task_run.run.execution_date,
is_root=task_run.is_root,
)
