def update_value_meta_conf_from_runtime_value(self, value, tracking_config):
# type: (Any, TrackingConfig) -> ParameterDefinition
return self.modify(
value_meta_conf=tracking_config.get_value_meta_conf(
self.value_meta_conf, get_value_type_of_obj(value, ValueType())
)
)
def build_user_parameter_value(name, value, source):
"""
Build parameter value for user defined name and value
"""
value_type = get_value_type_of_obj(
value, default_value_type=DefaultObjectValueType)
param_f = get_parameter_for_value_type(value_type)
param = build_parameter(param_f)
param.name = name
if value is NOTHING:
parameter, warnings = param, []
actual_value = param.default
else:
parameter, warnings = infer_parameter_value_type(param, value)
actual_value = value
return ParameterValue(
parameter=parameter,
source=source,
source_value=value,
value=actual_value,
parsed=False,
warnings=warnings,
)
def _build_value_type(self, context):
s = self.parameter
value_type = s.value_type
default = s.default
if value_type is None:
if is_defined(default):
value_type = get_value_type_of_obj(default)
if value_type is None:
raise friendly_error.task_parameters.no_value_type_from_default(
default, context=context)
elif value_type is None: # we don't have value type! let's fail!
if s.load_on_build is False: # we are in data mode
s.value_type = TargetValueType
else:
raise friendly_error.task_parameters.no_value_type_defined_in_parameter(
context=context)
else:
# let validate that what we have can be ValueType!
resolved_value_type = get_value_type_of_type(value_type)
if resolved_value_type is None: # we don't have value type! let's fail!
raise friendly_error.task_parameters.unknown_value_type_in_parameter(
value_type)
value_type = resolved_value_type # type: ValueType
if s.sub_type:
sub_value_type = get_value_type_of_type(s.sub_type)
if isinstance(value_type, _StructureValueType):
value_type = value_type.__class__(
sub_value_type=sub_value_type)
else:
raise friendly_error.task_parameters.sub_type_with_non_structural_value(
context=context,
value_type=value_type,
sub_type=s.sub_type)
return value_type
def _log_parameter_value(task_run, parameter_definition, target, value):
# make sure it will be logged correctly
parameter_definition = parameter_definition.modify(
value_meta_conf=ValueMetaConf(
log_preview=True,
log_schema=True,
))
try:
# case what if result is Proxy
value_type = get_value_type_of_obj(value,
parameter_definition.value_type)
task_run.run.target_origin.add(target, value, value_type)
except Exception as ex:
log_exception("Failed to register result to target tracking.",
ex=ex,
non_critical=True)
try:
task_run.tracker.log_parameter_data(
parameter=
parameter_definition, # was: task_run.task.task_definition.task_class.result,
target=target,
value=value,
operation_type=DbndTargetOperationType.
write, # is it write? (or log?)
operation_status=DbndTargetOperationStatus.OK,
)
except Exception as ex:
log_exception("Failed to log result to tracking store.",
ex=ex,
non_critical=True)
def auto_save_param(self, parameter, original_value, current_value):
# type: (ParameterDefinition, Any, Any) -> None
# it's output! we are going to save it.
# task run doesn't always exist
task_run = try_get_current_task_run()
access_status = DbndTargetOperationStatus.OK
try:
if isinstance(original_value, InMemoryTarget):
parameter.value_type = get_value_type_of_obj(
current_value, parameter.value_type
)
parameter.dump_to_target(original_value, current_value)
# it's a workaround, we don't want to change parameter for outputs (dynamically)
# however, we need proper value type to "dump" preview an other meta.
# we will update it only for In memory targets only for now
except Exception as ex:
access_status = DbndTargetOperationStatus.NOK
raise friendly_error.task_execution.failed_to_save_value_to_target(
ex, self.task, parameter, original_value, current_value
)
finally:
if task_run:
try:
task_run.tracker.log_parameter_data(
parameter=parameter,
target=original_value,
value=current_value,
operation_type=DbndTargetOperationType.write,
operation_status=access_status,
)
except Exception as ex:
logger.warning("Failed to log target to tracking store. %s", ex)
def get_value_meta(value,
meta_conf,
tracking_config,
value_type=None,
target=None):
# type: ( Any, ValueMetaConf, TrackingConfig, Optional[ValueType], Optional[Target]) -> Optional[ValueMeta]
"""
Build the value meta for tracking logging.
Using the given meta config, the value, and tracking_config to calculate the required value meta.
@param value: the value to calc value meta for
@param meta_conf: a given meta_config by a user
@param tracking_config: TrackingConfig to calc the wanted meta conf
@param value_type: optional value_type, if its known.
@param target: knowledge about the target which contains the value - this can effect the cost of the calculation
@return: Calculated value meta
"""
if value is None:
return None
# required for calculating the relevant configuration and to build value_meta
if _is_default_value_type(value_type) or isinstance(
value_type, TargetValueType):
# we calculate the actual value_type even if the given value is the default value
# so we can be sure that we can report it the right way
# also Targets are futures types and now would can log their actual value
value_type = get_value_type_of_obj(
value, default_value_type=ObjectValueType())
meta_conf = tracking_config.get_value_meta_conf(meta_conf, value_type,
target)
return value_type.get_value_meta(value, meta_conf=meta_conf)
def _log_parameter_value(task_run, parameter_definition, target, value):
# type: (TaskRun, ParameterDefinition, Target, Any) -> None
# make sure it will be logged correctly
parameter_definition = (
parameter_definition.update_value_meta_conf_from_runtime_value(
value, task_run.run.context.settings.tracking))
try:
# case what if result is Proxy
value_type = get_value_type_of_obj(value,
parameter_definition.value_type)
task_run.run.target_origin.add(target, value, value_type)
except Exception as ex:
log_exception("Failed to register result to target tracking.",
ex=ex,
non_critical=True)
try:
task_run.tracker.log_parameter_data(
parameter=parameter_definition,
target=target,
value=value,
operation_type=DbndTargetOperationType.
write, # is it write? (or log?)
operation_status=DbndTargetOperationStatus.OK,
)
except Exception as ex:
log_exception("Failed to log result to tracking store.",
ex=ex,
non_critical=True)
def _log_parameter_value(task_run, parameter_definition, target, value):
try:
# case what if result is Proxy
value_type = get_value_type_of_obj(value,
parameter_definition.value_type)
task_run.run.target_origin.add(target, value, value_type)
except Exception as ex:
log_exception("Failed to register result to target tracking.",
ex=ex,
non_critical=True)
try:
task_run.tracker.log_parameter_data(
parameter=
parameter_definition, # was: task_run.task.task_definition.task_class.result,
target=target,
value=value,
operation_type=DbndTargetOperationType.
write, # is it write? (or log?)
operation_status=DbndTargetOperationStatus.OK,
)
except Exception as ex:
log_exception("Failed to log result to tracking store.",
ex=ex,
non_critical=True)
def __init__(self, obj=_NOTHING, path=None, value_type=None, **kwargs):
from targets.values import get_value_type_of_obj
super(InMemoryTarget, self).__init__(**kwargs)
self._obj = obj
if self._obj is _NOTHING and not path:
raise Exception("InMemoryTarget requires object or path")
self.value_type = value_type or get_value_type_of_obj(self._obj)
self.path = path or "memory://%s:%s" % (
self.value_type,
self.value_type.to_signature(self._obj),
)
def safe_value_to_str(value, max_len):
if value is None or isinstance(value, six.string_types):
return safe_short_string(str(value), max_len)
try:
# local import to prevent imports recursion
from targets.values import get_value_type_of_obj
value_type = get_value_type_of_obj(value)
if value_type is not None:
return value_type.to_preview(value, max_len)
except Exception:
logger.warning("failed dumping metric, falling back to str", exc_info=True)
# This object has to hold only serializable values
return safe_short_string(str(value), max_len)
def dump(self, value, value_type=None, **kwargs):
from targets.values import get_value_type_of_obj, get_value_type_of_type
from targets.values import ObjectValueType
if value_type:
value_type = get_value_type_of_type(value_type)
else:
value_type = get_value_type_of_obj(value, ObjectValueType())
try:
m = get_marshaller_ctrl(self, value_type=value_type)
m.dump(value, **kwargs)
except Exception as ex:
raise friendly_error.failed_to_write_task_output(
ex, self, value_type=value_type
)
cache_key = TargetCacheKey(target=self, value_type=value_type)
TARGET_CACHE[cache_key] = value
def get_value_meta_for_metric(key, value, with_preview=True, with_data_hash=False):
value_type = get_value_type_of_obj(value)
if value_type is None:
logger.info(
"Can't detect known type for '%s' with type='%s' ", key, type(value)
)
return None
try:
data_dimensions = value_type.get_data_dimensions(value)
if data_dimensions is not None:
data_dimensions = list(data_dimensions)
try:
preview = value_type.to_preview(value) if with_preview else None
data_hash = value_type.get_data_hash(value) if with_data_hash else None
except Exception as ex:
logger.info(
"Failed to calculate data preview for '%s' with type='%s'"
" ( detected as %s): %s",
key,
type(value),
value_type,
ex,
)
data_hash = preview = None
data_schema = value_type.get_data_schema(value)
return TargetMeta(
value_preview=preview,
data_dimensions=data_dimensions,
data_schema=data_schema,
data_hash=data_hash,
)
except Exception as ex:
logger.info(
"Failed to build value meta info for '%s' with type='%s'"
" ( detected as %s): %s",
key,
type(value),
value_type,
ex,
)
return None
def get_value_meta_for_metric(key, value, meta_conf):
value_type = get_value_type_of_obj(value)
if value_type is None:
logger.info("Can't detect known type for '%s' with type='%s' ", key,
type(value))
return None
try:
return value_type.get_value_meta(value, meta_conf=meta_conf)
except Exception as ex:
logger.info(
"Failed to get value meta info for '%s' with type='%s'"
" ( detected as %s): %s",
key,
type(value),
value_type,
ex,
)
return None
def dump(self, value, value_type=None, **kwargs):
self._obj = value
if value_type:
self.value_type = get_value_type_of_type(value_type)
else:
self.value_type = get_value_type_of_obj(value, ObjectValueType())
