def update_value_meta_conf_from_runtime_value(self, value, tracking_config): # type: (Any, TrackingConfig) -> ParameterDefinition return self.modify( value_meta_conf=tracking_config.get_value_meta_conf( self.value_meta_conf, get_value_type_of_obj(value, ValueType()) ) )
def build_user_parameter_value(name, value, source): """ Build parameter value for user defined name and value """ value_type = get_value_type_of_obj( value, default_value_type=DefaultObjectValueType) param_f = get_parameter_for_value_type(value_type) param = build_parameter(param_f) param.name = name if value is NOTHING: parameter, warnings = param, [] actual_value = param.default else: parameter, warnings = infer_parameter_value_type(param, value) actual_value = value return ParameterValue( parameter=parameter, source=source, source_value=value, value=actual_value, parsed=False, warnings=warnings, )
def _build_value_type(self, context): s = self.parameter value_type = s.value_type default = s.default if value_type is None: if is_defined(default): value_type = get_value_type_of_obj(default) if value_type is None: raise friendly_error.task_parameters.no_value_type_from_default( default, context=context) elif value_type is None: # we don't have value type! let's fail! if s.load_on_build is False: # we are in data mode s.value_type = TargetValueType else: raise friendly_error.task_parameters.no_value_type_defined_in_parameter( context=context) else: # let validate that what we have can be ValueType! resolved_value_type = get_value_type_of_type(value_type) if resolved_value_type is None: # we don't have value type! let's fail! raise friendly_error.task_parameters.unknown_value_type_in_parameter( value_type) value_type = resolved_value_type # type: ValueType if s.sub_type: sub_value_type = get_value_type_of_type(s.sub_type) if isinstance(value_type, _StructureValueType): value_type = value_type.__class__( sub_value_type=sub_value_type) else: raise friendly_error.task_parameters.sub_type_with_non_structural_value( context=context, value_type=value_type, sub_type=s.sub_type) return value_type
def _log_parameter_value(task_run, parameter_definition, target, value): # make sure it will be logged correctly parameter_definition = parameter_definition.modify( value_meta_conf=ValueMetaConf( log_preview=True, log_schema=True, )) try: # case what if result is Proxy value_type = get_value_type_of_obj(value, parameter_definition.value_type) task_run.run.target_origin.add(target, value, value_type) except Exception as ex: log_exception("Failed to register result to target tracking.", ex=ex, non_critical=True) try: task_run.tracker.log_parameter_data( parameter= parameter_definition, # was: task_run.task.task_definition.task_class.result, target=target, value=value, operation_type=DbndTargetOperationType. write, # is it write? (or log?) operation_status=DbndTargetOperationStatus.OK, ) except Exception as ex: log_exception("Failed to log result to tracking store.", ex=ex, non_critical=True)
def auto_save_param(self, parameter, original_value, current_value): # type: (ParameterDefinition, Any, Any) -> None # it's output! we are going to save it. # task run doesn't always exist task_run = try_get_current_task_run() access_status = DbndTargetOperationStatus.OK try: if isinstance(original_value, InMemoryTarget): parameter.value_type = get_value_type_of_obj( current_value, parameter.value_type ) parameter.dump_to_target(original_value, current_value) # it's a workaround, we don't want to change parameter for outputs (dynamically) # however, we need proper value type to "dump" preview an other meta. # we will update it only for In memory targets only for now except Exception as ex: access_status = DbndTargetOperationStatus.NOK raise friendly_error.task_execution.failed_to_save_value_to_target( ex, self.task, parameter, original_value, current_value ) finally: if task_run: try: task_run.tracker.log_parameter_data( parameter=parameter, target=original_value, value=current_value, operation_type=DbndTargetOperationType.write, operation_status=access_status, ) except Exception as ex: logger.warning("Failed to log target to tracking store. %s", ex)
def get_value_meta(value, meta_conf, tracking_config, value_type=None, target=None): # type: ( Any, ValueMetaConf, TrackingConfig, Optional[ValueType], Optional[Target]) -> Optional[ValueMeta] """ Build the value meta for tracking logging. Using the given meta config, the value, and tracking_config to calculate the required value meta. @param value: the value to calc value meta for @param meta_conf: a given meta_config by a user @param tracking_config: TrackingConfig to calc the wanted meta conf @param value_type: optional value_type, if its known. @param target: knowledge about the target which contains the value - this can effect the cost of the calculation @return: Calculated value meta """ if value is None: return None # required for calculating the relevant configuration and to build value_meta if _is_default_value_type(value_type) or isinstance( value_type, TargetValueType): # we calculate the actual value_type even if the given value is the default value # so we can be sure that we can report it the right way # also Targets are futures types and now would can log their actual value value_type = get_value_type_of_obj( value, default_value_type=ObjectValueType()) meta_conf = tracking_config.get_value_meta_conf(meta_conf, value_type, target) return value_type.get_value_meta(value, meta_conf=meta_conf)
def _log_parameter_value(task_run, parameter_definition, target, value): # type: (TaskRun, ParameterDefinition, Target, Any) -> None # make sure it will be logged correctly parameter_definition = ( parameter_definition.update_value_meta_conf_from_runtime_value( value, task_run.run.context.settings.tracking)) try: # case what if result is Proxy value_type = get_value_type_of_obj(value, parameter_definition.value_type) task_run.run.target_origin.add(target, value, value_type) except Exception as ex: log_exception("Failed to register result to target tracking.", ex=ex, non_critical=True) try: task_run.tracker.log_parameter_data( parameter=parameter_definition, target=target, value=value, operation_type=DbndTargetOperationType. write, # is it write? (or log?) operation_status=DbndTargetOperationStatus.OK, ) except Exception as ex: log_exception("Failed to log result to tracking store.", ex=ex, non_critical=True)
def _log_parameter_value(task_run, parameter_definition, target, value): try: # case what if result is Proxy value_type = get_value_type_of_obj(value, parameter_definition.value_type) task_run.run.target_origin.add(target, value, value_type) except Exception as ex: log_exception("Failed to register result to target tracking.", ex=ex, non_critical=True) try: task_run.tracker.log_parameter_data( parameter= parameter_definition, # was: task_run.task.task_definition.task_class.result, target=target, value=value, operation_type=DbndTargetOperationType. write, # is it write? (or log?) operation_status=DbndTargetOperationStatus.OK, ) except Exception as ex: log_exception("Failed to log result to tracking store.", ex=ex, non_critical=True)
def __init__(self, obj=_NOTHING, path=None, value_type=None, **kwargs): from targets.values import get_value_type_of_obj super(InMemoryTarget, self).__init__(**kwargs) self._obj = obj if self._obj is _NOTHING and not path: raise Exception("InMemoryTarget requires object or path") self.value_type = value_type or get_value_type_of_obj(self._obj) self.path = path or "memory://%s:%s" % ( self.value_type, self.value_type.to_signature(self._obj), )
def safe_value_to_str(value, max_len): if value is None or isinstance(value, six.string_types): return safe_short_string(str(value), max_len) try: # local import to prevent imports recursion from targets.values import get_value_type_of_obj value_type = get_value_type_of_obj(value) if value_type is not None: return value_type.to_preview(value, max_len) except Exception: logger.warning("failed dumping metric, falling back to str", exc_info=True) # This object has to hold only serializable values return safe_short_string(str(value), max_len)
def dump(self, value, value_type=None, **kwargs): from targets.values import get_value_type_of_obj, get_value_type_of_type from targets.values import ObjectValueType if value_type: value_type = get_value_type_of_type(value_type) else: value_type = get_value_type_of_obj(value, ObjectValueType()) try: m = get_marshaller_ctrl(self, value_type=value_type) m.dump(value, **kwargs) except Exception as ex: raise friendly_error.failed_to_write_task_output( ex, self, value_type=value_type ) cache_key = TargetCacheKey(target=self, value_type=value_type) TARGET_CACHE[cache_key] = value
def get_value_meta_for_metric(key, value, with_preview=True, with_data_hash=False): value_type = get_value_type_of_obj(value) if value_type is None: logger.info( "Can't detect known type for '%s' with type='%s' ", key, type(value) ) return None try: data_dimensions = value_type.get_data_dimensions(value) if data_dimensions is not None: data_dimensions = list(data_dimensions) try: preview = value_type.to_preview(value) if with_preview else None data_hash = value_type.get_data_hash(value) if with_data_hash else None except Exception as ex: logger.info( "Failed to calculate data preview for '%s' with type='%s'" " ( detected as %s): %s", key, type(value), value_type, ex, ) data_hash = preview = None data_schema = value_type.get_data_schema(value) return TargetMeta( value_preview=preview, data_dimensions=data_dimensions, data_schema=data_schema, data_hash=data_hash, ) except Exception as ex: logger.info( "Failed to build value meta info for '%s' with type='%s'" " ( detected as %s): %s", key, type(value), value_type, ex, ) return None
def get_value_meta_for_metric(key, value, meta_conf): value_type = get_value_type_of_obj(value) if value_type is None: logger.info("Can't detect known type for '%s' with type='%s' ", key, type(value)) return None try: return value_type.get_value_meta(value, meta_conf=meta_conf) except Exception as ex: logger.info( "Failed to get value meta info for '%s' with type='%s'" " ( detected as %s): %s", key, type(value), value_type, ex, ) return None
def dump(self, value, value_type=None, **kwargs): self._obj = value if value_type: self.value_type = get_value_type_of_type(value_type) else: self.value_type = get_value_type_of_obj(value, ObjectValueType())