def test_unflatten_dict_to_nested(self):
inputs1 = {'a//b': 1, 'a//c': 2, 'e': 3}
must1 = {'a': {'b': 1, 'c': 2}, 'e': 3}
self.assertDictEqual(nest_utils.unflatten_dict_to_nested(inputs1),
must1)
inputs2 = {'a//b': 1, 'b': 2, 'e': 3}
must2 = {'a': {'b': 1}, 'b': 2, 'e': 3}
self.assertDictEqual(nest_utils.unflatten_dict_to_nested(inputs2),
must2)
inputs3 = {
'a//b': 1,
'a//c': 2,
'a//d//d1': 2,
'a//d//d2': 4,
'e//0': 1,
'e//1': 2,
'e//2//e1': 10
}
must3 = {
'a': {
'b': 1,
'c': 2,
'd': {
'd1': 2,
'd2': 4
}
},
'e': [1, 2, {
'e1': 10
}]
}
self.assertDictEqual(nest_utils.unflatten_dict_to_nested(inputs3),
must3)
def filter_kpi_values(kpi: dict,
return_flattened: bool = False) -> (dict, dict):
"""
Filter kpis according to its value type. If kpi value is not of type str
or number, it will be filtered out. If value is numpy array of size 1, then
element will be selected and not filtered out.
Parameters
----------
kpi
dict, possibly nested, mapping kpi names to its values
return_flattened
flag to return flattened dict and do not unflatten it back
Returns
-------
kpi_filtered
dict with same structure as kpi, but only with values of numbers and
string type
kpi_filtered_out
dict with same structure as kpi with values other then numbers and
string type
"""
logger = logging.getLogger(__name__)
kpi_flatten = nest_utils.flatten_nested_struct(kpi)
names_filtered_out = []
for kpi_name in kpi_flatten:
kpi_value = kpi_flatten[kpi_name]
if isinstance(kpi_value, np.ndarray) and np.prod(kpi_value.shape) == 1:
kpi_value = np.reshape(kpi_value, (1, ))[0]
# pylint: disable=no-member
# numpy does have floating member
if isinstance(kpi_value, np.floating):
kpi_value = float(kpi_value)
elif isinstance(kpi_value, np.integer):
kpi_value = int(kpi_value)
elif isinstance(kpi_value, np.str):
kpi_value = str(kpi_value)
kpi_flatten[kpi_name] = kpi_value
if not isinstance(kpi_value, (numbers.Number, str, list)):
names_filtered_out.append(kpi_name)
kpi_filtered = {
k: v
for k, v in kpi_flatten.items() if k not in names_filtered_out
}
kpi_filtered_out = {
k: v
for k, v in kpi_flatten.items() if k in names_filtered_out
}
if kpi_filtered and not return_flattened:
kpi_filtered = nest_utils.unflatten_dict_to_nested(kpi_filtered)
if kpi_filtered_out:
logger.warning(
"Following kpi keys cannot be serialized to json: "
"%s", kpi_filtered_out.keys())
if not return_flattened:
kpi_filtered_out = nest_utils.unflatten_dict_to_nested(
kpi_filtered_out)
return kpi_filtered, kpi_filtered_out
def split_batch_inputs(
inputs: dict,
not_batch_keys: Optional[List[str]] = None,
ignore_none_values=True,
) -> Tuple[List[dict], dict]:
"""
Split batch inputs to sample inputs
Parameters
----------
inputs
batch inputs to split
not_batch_keys
keys to exclude from split
ignore_none_values
if the keys with None values should be treated as not batch keys
Returns
-------
batch_inputs_flat_as_list
list split batch inputs
not_batch_inputs
dict with not batch inputs
"""
not_batch_keys = not_batch_keys or []
batch_inputs = {
each_key: each_value
for each_key, each_value in inputs.items()
if each_key not in not_batch_keys
}
not_batch_inputs = {
each_key: each_value
for each_key, each_value in inputs.items()
if each_key in not_batch_keys
}
batch_inputs_flat = nest_utils.flatten_nested_struct(batch_inputs)
if ignore_none_values:
none_keys = [k for k, v in batch_inputs_flat.items() if v is None]
batch_inputs_flat = {
k: v
for k, v in batch_inputs_flat.items() if k not in none_keys
}
not_batch_inputs.update(
nest_utils.unflatten_dict_to_nested({k: None
for k in none_keys}))
batch_inputs_flat_as_list = (
nest_utils.dict_of_lists_to_list_of_dicts(batch_inputs_flat))
batch_inputs_as_list = [
nest_utils.unflatten_dict_to_nested(each_flat_input)
for each_flat_input in batch_inputs_flat_as_list
]
return batch_inputs_as_list, not_batch_inputs
def represent_predictor_through_nucleotides(
predictor: Predictor,
incoming_keys_mapping: Optional[dict] = None) -> List[Nucleotide]:
"""
Represent predictor feed tensors structure as list of nucleotides
When predictor has following fetch'_tensors structure but in flatten
`{'postprocessor_nucleotide1': {'out1': ..., 'out2': ...},
'postprocessor_nucleotide2': {'out3': ..., 'out4': ...}}`,
then it will return list of nodes
`[postprocessor_nucleotide1, postprocessor_nucleotide2]` with generated keys
`['out1', 'out2']` and `['out3', 'out4']` respectively
Parameters
----------
predictor
predictor with fetch_tensors property defined
incoming_keys_mapping
mapping of the incoming keys to the predictor
Returns
-------
predictor_nucleotides
list of nucleotides which mimic the data structure of fetched predictor
tensors
"""
fetch_tensors_flatten = predictor.fetch_tensors
fetch_tensors = nest_utils.unflatten_dict_to_nested(fetch_tensors_flatten)
feeded_tensors_without_parameters_flatten = {
each_key: each_tensor
for each_key, each_tensor in predictor.feed_tensors.items()
if each_tensor.op.type != 'PlaceholderWithDefault'
}
feeded_tensors_without_parameters = nest_utils.unflatten_dict_to_nested(
feeded_tensors_without_parameters_flatten)
incoming_keys = sorted(list(feeded_tensors_without_parameters))
predictor_nucleotides = []
for each_nucleotide_name_key, each_nucleotide_outputs in sorted(
fetch_tensors.items()):
nucleotide = PredictorNucleotide(name=each_nucleotide_name_key,
inbound_nodes="dataset")
nucleotide.incoming_keys = incoming_keys
if incoming_keys_mapping is not None:
nucleotide.incoming_keys_mapping = {
"dataset": incoming_keys_mapping
}
nucleotide.generated_keys = sorted(list(each_nucleotide_outputs))
predictor_nucleotides.append(nucleotide)
return predictor_nucleotides
def remap_single_input(inputs: dict, mapping: Optional[dict] = None) -> dict:
"""
Remap single input keys according to mapping
Parameters
----------
inputs
dict with inputs, where keys should be remapped
mapping
mapping of old keys to new keys; if some key was not present, it will
be passed as is; if new key is "_", it will be ignored in remapped
result
Returns
-------
remapped_inputs
inputs with remapped keys
"""
inputs_remapped_flat = {}
mapping = mapping or {}
inputs_flat = nest_utils.flatten_nested_struct(
inputs, separator=_NESTED_KEY_SEPARATOR)
for old_name, value in sorted(inputs_flat.items()):
remapped_new_names = _get_new_key_for_nested_input_and_map(
old_name, mapping)
for each_new_name in remapped_new_names:
if each_new_name == NucleotideKeyFields.IGNORE_KEY:
continue
inputs_remapped_flat[each_new_name] = value
inputs_remapped = (nest_utils.unflatten_dict_to_nested(
inputs_remapped_flat, separator=_NESTED_KEY_SEPARATOR))
return inputs_remapped
def _get_nested_shapes(nested_dict):
flatten = nest_utils.flatten_nested_struct(nested_dict)
flatten_with_shapes = {k: v.get_shape().as_list()
for k, v in flatten.items()}
nested_with_shapes = nest_utils.unflatten_dict_to_nested(
flatten_with_shapes)
return nested_with_shapes
def select_inputs_by_sample_mask_np(sample_mask: np.ndarray,
keys_to_exclude_from_sample_mask: Optional[
List[str]] = None,
**inputs) -> Dict[str, np.ndarray]:
"""
Select inputs by masking out samples with sample_mask == 0
Parameters
----------
sample_mask
tensor of shape [batch_size] with 1 indicating that sample should
be leaved as is and 0 - remove sample
keys_to_exclude_from_sample_mask
list of keys that will not be masked using sample_mask
**inputs
inputs to mask
Returns
-------
masked_inputs
masked inputs sample-wise
"""
inputs_flatten = nest_utils.flatten_nested_struct(inputs)
inputs_masked_flatten = {}
keys_to_exclude = keys_to_exclude_from_sample_mask or []
sample_mask = sample_mask.astype(bool)
for each_key, each_value in inputs_flatten.items():
if each_key in keys_to_exclude:
inputs_masked_flatten[each_key] = each_value
else:
inputs_masked_flatten[each_key] = each_value[sample_mask]
inputs_masked = nest_utils.unflatten_dict_to_nested(inputs_masked_flatten)
return inputs_masked
def _get_data_results(data: tf.data.Dataset, session_manager,
max_iteration=None) -> dict:
iterator = data.make_one_shot_iterator()
sample = iterator.get_next()
outputs_flatten = {}
iteration_number = 0
with session_manager as sess:
while True:
try:
sample_out = sess.run(sample)
sample_out_flatten = nest_utils.flatten_nested_struct(
sample_out)
for k, v in sample_out_flatten.items():
outputs_flatten.setdefault(k, [])
if isinstance(v, bytes):
v = v.decode()
outputs_flatten[k].append(v)
iteration_number += 1
except tf.errors.OutOfRangeError:
break
if max_iteration is not None and iteration_number >= max_iteration:
break
outputs = nest_utils.unflatten_dict_to_nested(outputs_flatten)
return outputs
def calculate_losses(
self, *, inputs_from_dataset: _NESTED_TENSORS_DTYPE,
predictions_raw: _NESTED_TENSORS_DTYPE
) -> Dict[str, Union[Dict[str, tf.Tensor], tf.Tensor]]:
"""
Create loss for model using labels from 'inputs'
and predictions
Parameters
----------
inputs_from_dataset
dictionary holding the dataset inputs
predictions_raw
dict with prediction tensors as values
Returns
-------
losses
dict with values as losses; inside of keys should be 'total_loss'
with value to optimize; default ragularizations on the all
training variables will be applied afterwards if specified
"""
gene_inputs = {}
gene_inputs.update(inputs_from_dataset)
gene_inputs.update(predictions_raw)
losses = self._process_gene(gene_name='losses',
gene_inputs=gene_inputs)
unflatten_losses = nest_utils.unflatten_dict_to_nested(losses)
total_loss = 0.0
for loss in unflatten_losses.values():
if 'total_loss' in loss:
total_loss += loss['total_loss']
losses['total_loss'] = total_loss
losses = self._add_regularization(losses)
return losses
def preprocess_dataset_inputs(
inputs: Dict[str, tf.Tensor]) -> Dict[str, Dict[str, tf.Tensor]]:
"""
Add preprocessing step as identity nodes on dataset inputs
and add all the inputs to dataset key
Parameters
----------
inputs
inputs from datasets
Returns
-------
inputs_with_identity
same as inputs, but with added identity ops and add to dataset key
"""
inputs_flat = nest_utils.flatten_nested_struct(inputs)
inputs_flat_identity = {
k: tf.identity(v)
for k, v in sorted(inputs_flat.items())
}
inputs_identity = nest_utils.unflatten_dict_to_nested(
inputs_flat_identity)
inputs_identity = {"dataset": inputs_identity}
return inputs_identity
def predict(self, **inputs) -> Dict[str, tf.Tensor]:
result_flat = {}
default_axis = self.axis.get("default")
inputs_flat = nest_utils.flatten_nested_struct(inputs,
flatten_lists=False)
for each_key, each_input_list in inputs_flat.items():
if not isinstance(each_input_list, (list, tuple)):
msg = ("{}: all inputs to concat must be lists or tuples! "
"(input for key {} is of type {})").format(
self.name, each_key, type(each_input_list))
raise ValueError(msg)
axis_for_key = self.axis.get(each_key, default_axis)
if axis_for_key is None:
msg = ("{}: axis for key {} was not provided and default key "
"does not exist!").format(self.name, each_key)
raise ValueError(msg)
if axis_for_key >= len(each_input_list[0].shape):
msg = ("{}: axis {} for input key {} is not valid for"
"inputs with shape {}").format(self.name, axis_for_key,
each_key,
each_input_list[0].shape)
raise ValueError(msg)
inputs_concat = tf.concat(each_input_list, axis=axis_for_key)
result_flat[each_key] = inputs_concat
result = nest_utils.unflatten_dict_to_nested(result_flat)
return result
def predict_batch(self, inputs: Union[dict, list]) -> Tuple[dict, float]:
"""
Make predictions given inputs
Parameters
----------
inputs
inputs to the network
Returns
-------
predictions
predictions of the network
predict_exec_time
execution time of network prediction
"""
time_start_predict = time.time()
if not isinstance(inputs, list):
inputs = [inputs]
list_of_predictions = []
for each_input in inputs:
if self.model_incoming_keys_mapping is not None:
each_input = nucleotide_utils.remap_and_collapse_inputs(
[each_input], [self.model_incoming_keys_mapping])
each_input_flatten = nest_utils.flatten_nested_struct(each_input)
current_prediction_flatten = predictors.predict_using_predictor(
predictor=self._predictor,
inputs=each_input_flatten,
model_parameters=self.model_parameters)
list_of_predictions.append(current_prediction_flatten)
predictions_flatten = nucleotide_utils.collapse_inputs(
list_of_predictions)
predictions = nest_utils.unflatten_dict_to_nested(predictions_flatten)
predict_exec_time = time.time() - time_start_predict
return predictions, predict_exec_time
def combine_fn(*list_of_features) -> tf.data.Dataset:
"""
Method to combine the features
Parameters
----------
list_of_features
list of features to combine
Returns
-------
data_with_combined_features
data with combined features
"""
features_combined_flatten = {}
for each_features in list_of_features:
each_features_flatten = nest_utils.flatten_nested_struct(
each_features)
for (each_feature_name,
each_feature) in each_features_flatten.items():
if each_feature_name in features_combined_flatten:
_assert_tensors_have_same_shape(
features_combined_flatten[each_feature_name],
each_feature)
_assert_tensors_have_same_type(
features_combined_flatten[each_feature_name],
each_feature)
else:
features_combined_flatten[each_feature_name] = each_feature
features_combined = nest_utils.unflatten_dict_to_nested(
features_combined_flatten)
data_with_featured = tf.data.Dataset.from_tensors(features_combined)
return data_with_featured
def maybe_cast_dtype(
inputs: Dict[str, tf.Tensor],
cast_dtypes: Dict[tf.DType, tf.DType] = None) -> Dict[str, tf.Tensor]:
"""
Cast values from nested inputs structure according to cast_dtypes mapping.
If dtype of value inside of inputs is not inside of cast_dtypes keys, it
will not be casted at all.
Parameters
----------
inputs
possibly nested dict, with values as tensors
cast_dtypes
dict with mapping of which dtype should be casted to which, e.g.
{float32: float16} means that all of float32 tensors will be casted
to float16 before passing to nucleotide
Returns
-------
inputs_casted : dict
same structure as inputs, but with inputs casted according to
cast_dtypes
"""
if cast_dtypes is None:
return inputs
inputs_flatten = nest_utils.flatten_nested_struct(inputs)
for k, each_input in inputs_flatten.items():
if not isinstance(each_input, tf.Tensor):
continue
dtype_input = each_input.dtype
if dtype_input in cast_dtypes:
input_casted = tf.cast(each_input, cast_dtypes[dtype_input])
inputs_flatten[k] = input_casted
inputs_casted = nest_utils.unflatten_dict_to_nested(inputs_flatten)
return inputs_casted
def _get_default_features(nested_features):
features_flatten = nest_utils.flatten_nested_struct(nested_features)
zero_values_flatten = {
each_key: tf.zeros_like(each_value)
for each_key, each_value in features_flatten.items()
}
zero_values = nest_utils.unflatten_dict_to_nested(zero_values_flatten)
return zero_values
def get(self) -> dict:
"""
Returns
-------
buffer_dict
nested dict with buffer
"""
return nest_utils.unflatten_dict_to_nested(self._buffer_flat)
def _format_data_for_log(data: Optional[dict] = None,
shape_fn: Callable = np.shape) -> Optional[dict]:
if data is None:
return None
data_flat = nest_utils.flatten_nested_struct(data)
data_repr_flat = {
k: (v if isinstance(v, tf.Tensor) else shape_fn(v))
for k, v in data_flat.items()
}
data_repr = nest_utils.unflatten_dict_to_nested(data_repr_flat)
return data_repr
def get_summaries(
self, *, inputs_from_dataset: _NESTED_TENSORS_DTYPE,
predictions_raw: _NESTED_TENSORS_DTYPE,
predictions: _NESTED_TENSORS_DTYPE) -> _NESTED_TENSORS_DTYPE:
"""
Create the summaries for model as dict
to separate different types of summaries, the prefix is used:
- `scalar_{}`
- `image_{}`
- `histogram_{}`
- `text_{}`
- `audio_{}`
Names without this prefixes will not be stored to tensorboard.
Parameters
----------
inputs_from_dataset
dictionary holding the dataset inputs
predictions_raw
dict with raw predictions
predictions
dict with output tensors as values
Returns
-------
summaries
combined summaries from all `ModelSummary` instances
"""
gene_inputs = {}
gene_inputs.update(inputs_from_dataset)
gene_inputs.update(predictions_raw)
gene_inputs.update(predictions)
summaries = self._process_gene(gene_name='summaries',
gene_inputs=gene_inputs)
summaries_not_to_store = [
n for n, s in self.summaries.items()
if not s.store_inside_tensorboard
]
if summaries_not_to_store:
logger = logging.getLogger(__name__)
logger.info(
'Summaries with names %s will not be stored to '
'tensorboard', summaries_not_to_store)
summaries_unfatten = nest_utils.unflatten_dict_to_nested(summaries)
for each_summary_name in summaries_not_to_store:
del summaries_unfatten[each_summary_name]
summaries = nest_utils.flatten_nested_struct(summaries_unfatten)
return summaries
def collection2nested(collection_prefix: str,
separator: str = '::',
graph: Optional[tf.Graph] = None,
raise_error: bool = True) -> dict:
"""
Construct the dict with keys as suffixes of collection
names and values as corresponding tensors / variables / ops
or list / value depending on the found collection_list
Parameters
----------
collection_prefix
prefix to collection name
separator
separator to separate the suffix from key
graph
graph to add the collections
raise_error
if ValueError should be raised if no collections with defined prefix
were found
Returns
-------
dict_with_values
dict with values from collections
"""
graph = _maybe_get_default_graph(graph)
collection_list = get_collections_by_prefix(
collection_prefix, separator, graph, raise_error=raise_error)
if not collection_list:
return {}
if len(collection_list) == 1 and collection_list[0] == collection_prefix:
return graph.get_collection(collection_prefix)[0]
result = {}
for collection_name in collection_list:
if len(graph.get_collection(collection_name)) == 1:
value = graph.get_collection(collection_name)[0]
else:
value = graph.get_collection(collection_name)
name_splitted = collection_name.split(separator)
key = name_splitted[1]
result[key] = value
if len(result) == 1 and list(result.keys())[0] == "":
result = list(result.values())[0]
result = nest_utils.unflatten_dict_to_nested(result)
return result
def restore(self):
io_utils.maybe_mkdir(self.cache_target)
cache_fname = self._get_cache_fname()
if not os.path.exists(cache_fname):
return None
with open(cache_fname, "r") as file:
restored = json.load(file)
for each_key in restored:
if isinstance(restored[each_key], list):
restored[each_key] = np.array(restored[each_key])
restored_unflatten = nest_utils.unflatten_dict_to_nested(restored)
logger = logging.getLogger(__name__)
logger.debug("restoring KPI values from %s", cache_fname)
return restored_unflatten
def parse_tfrecord_example(self, example) -> dict:
"""Parse tfrecord example"""
features_flat = nest_utils.flatten_nested_struct(
self.get_tfrecords_features(), '/')
output_types = self.get_tfrecords_output_types() or {}
output_types_flat = nest_utils.flatten_nested_struct(
output_types, '/')
parsed_example = tf.parse_single_example(example, features_flat)
data_decoded = {}
for field_name, field_value in parsed_example.items():
output_type = output_types_flat.get(field_name)
data_decoded[field_name] = self.decode_field(
field_name, field_value, output_type)
data = nest_utils.unflatten_dict_to_nested(data_decoded, '/')
data = self.postprocess_tfrecords(**data)
return data
def combine_samples_to_batch(
self,
list_of_sample_results: List[Optional[dict]]) -> Optional[dict]:
"""
Combine sample data to batch
Parameters
----------
list_of_sample_results
list of sample results
Returns
-------
batch_result
batch result
"""
# pylint: disable=no-self-use
# is an interface
list_of_sample_results_valid = [
each_result for each_result in list_of_sample_results
if each_result is not None
]
if not list_of_sample_results_valid:
return None
list_of_flat_sample_results = [
nest_utils.flatten_nested_struct(each_result)
for each_result in list_of_sample_results_valid
]
result_flat = {}
for each_key in list_of_flat_sample_results[0]:
result_flat[each_key] = np_utils.stack_with_pad([
each_result[each_key]
for each_result in list_of_flat_sample_results
])
result = nest_utils.unflatten_dict_to_nested(result_flat)
return result
def test_create_batch(self, mode, fix_batch_dimension):
self.datasets_for_mix[0].fix_batch_dimension = fix_batch_dimension
self.datasets_for_mix[1].fix_batch_dimension = fix_batch_dimension
dataset_mix = DatasetMix(datasets=self.datasets_for_mix).build()
dataset_mix.mode = mode
batch_size = 10
data_batch = dataset_mix.create_batch(batch_size=batch_size)
batch_dim_must = batch_size if fix_batch_dimension else None
output_shapes_must = {'input1': [batch_dim_must],
'input2': [batch_dim_must],
'input3': {'int': [batch_dim_must, 20]},
'input4': [batch_dim_must],
'sample_mask_data1': [batch_dim_must],
'sample_mask_data2': [batch_dim_must]}
output_types_must = {'input1': tf.string,
'input2': tf.float32,
'input3': {'int': tf.int32},
'input4': tf.string,
'sample_mask_data1': tf.float32,
'sample_mask_data2': tf.float32}
output_shapes_list_flatten = nest_utils.flatten_nested_struct(
data_batch.output_shapes)
output_shapes_list_flatten = {
k: v.as_list() for k, v in output_shapes_list_flatten.items()}
output_shapes_list = nest_utils.unflatten_dict_to_nested(
output_shapes_list_flatten)
self.assertDictEqual(output_shapes_must,
output_shapes_list)
self.assertDictEqual(output_types_must,
data_batch.output_types)
_ = _get_data_results(data_batch, self.test_session(),
max_iteration=100)
def test_create_features_for_single_sample(self, mode,
sampling_weights=None):
dataset_mix = DatasetMix(datasets=self.datasets_for_mix,
sampling_weights=sampling_weights).build()
dataset_mix.mode = mode
data_mixed = dataset_mix.create_features_for_single_sample()
output_shapes_must = {'input1': [],
'input2': [],
'input3': {'int': [20]},
'input4': [],
'sample_mask_data1': [],
'sample_mask_data2': []}
output_types_must = {'input1': tf.string,
'input2': tf.float32,
'input3': {'int': tf.int32},
'input4': tf.string,
'sample_mask_data1': tf.float32,
'sample_mask_data2': tf.float32}
output_shapes_list_flatten = nest_utils.flatten_nested_struct(
data_mixed.output_shapes)
output_shapes_list_flatten = {
k: v.as_list() for k, v in output_shapes_list_flatten.items()}
output_shapes_list = nest_utils.unflatten_dict_to_nested(
output_shapes_list_flatten)
self.assertDictEqual(output_shapes_must,
output_shapes_list)
self.assertDictEqual(output_types_must,
data_mixed.output_types)
default_values_1 = {
'input3': {'int': np.zeros([self.number_of_samples1, 20],
dtype=np.int32)},
'input4': ["" for _ in range(self.number_of_samples1)],
'sample_mask_data1': [1.0] * self.number_of_samples1,
'sample_mask_data2': [0.0] * self.number_of_samples1
}
default_values_2 = {
'input2': [0.0 for _ in range(self.number_of_samples2)],
'sample_mask_data1': [0.0] * self.number_of_samples1,
'sample_mask_data2': [1.0] * self.number_of_samples1
}
output1_with_defaults_must = copy.deepcopy(self.inputs1)
output1_with_defaults_must.update(default_values_1)
output2_with_defaults_must = copy.deepcopy(self.inputs2)
output2_with_defaults_must.update(default_values_2)
if mode == 'train':
iterations_number = 500
else:
iterations_number = 100
output = _get_data_results(data_mixed, self.test_session(),
max_iteration=iterations_number)
if mode == 'eval':
outputs_must = {}
for each_key in ['input1', 'input2', 'input4',
'sample_mask_data1', 'sample_mask_data2']:
outputs_must[each_key] = _interleave(
output1_with_defaults_must[each_key],
output2_with_defaults_must[each_key],
iterations_number
)
outputs_must['input3'] = {
'int': _interleave(
output1_with_defaults_must['input3']['int'],
output2_with_defaults_must['input3']['int'],
iterations_number
)
}
for each_key in ['input2', 'input3', 'sample_mask_data1',
'sample_mask_data2']:
self.assertAllClose(outputs_must[each_key],
output[each_key])
for each_key in ['input1', 'input4']:
self.assertListEqual(outputs_must[each_key],
output[each_key])
else:
number_samples_per_dataset = [0, 0]
datasets_with_defaults = [output1_with_defaults_must,
output2_with_defaults_must]
for i_sample in range(iterations_number):
sample = _select_sample_at_index(output, i_sample)
dataset_ind = _get_dataset_id_from_sample(
sample, datasets_with_defaults)
number_samples_per_dataset[dataset_ind] += 1
sampling_weights_output = [v / iterations_number
for v in number_samples_per_dataset]
if sampling_weights is not None:
sampling_weights_norm_must = [v / sum(sampling_weights)
for v in sampling_weights]
else:
sampling_weights_norm_must = [0.5, 0.5]
atol = 10 / iterations_number
self.assertAllClose(sampling_weights_norm_must,
sampling_weights_output,
atol=atol)
def test_predict_using_predictor(self, with_model_parameters):
class PredictorMock(object):
def __init__(self_, fetch_tensors, feed_tensors):
self_.fetch_tensors = fetch_tensors
self_.feed_tensors = feed_tensors
def __call__(self_, inputs: dict):
return {k + '_out': v for k, v in inputs.items()}
data = {
'node1': {
'out1': 10,
'out2': 20
},
'node2': {
'out3': 30,
'out4': 40
},
'node3': {
'out5': 50
}
}
data_for_predictor = {
k: v
for k, v in data.items() if k in ['node1', 'node2']
}
if with_model_parameters:
model_parameters = {
"nucleotide1": {
"parameter1": 10
},
"nucleotide3": {
"parameter2": 20,
"parameter3": [30, 40]
}
}
else:
model_parameters = None
data_for_predictor_flatten = nest_utils.flatten_nested_struct(
data_for_predictor, flatten_lists=False)
feed_tensors = nest_utils.flatten_nested_struct(data_for_predictor,
flatten_lists=False)
predictor_out_flatten_must = {
k + '_out': v
for k, v in data_for_predictor_flatten.items()
}
predictor_out_must = nest_utils.unflatten_dict_to_nested(
predictor_out_flatten_must)
if with_model_parameters:
predictor_out_must.update({
"nucleotide1": {
"parameter1_out": 10
},
"nucleotide3": {
"parameter2_out": 20,
"parameter3_out": [30, 40]
}
})
predictor = PredictorMock(feed_tensors=feed_tensors,
fetch_tensors=None)
if with_model_parameters:
result = predict_using_predictor(predictor,
inputs=data,
model_parameters=model_parameters)
else:
result = predict_using_predictor(predictor, inputs=data)
self.assertDictEqual(predictor_out_must, result)
def predictor_from_load_config(
load_config: InferenceLoadConfig,
tensorrt_config: Optional[TensorrtConfig] = None,
session_config: Optional[tf.ConfigProto] = None,
postprocessors_to_use: Optional[List[str]] = None,
model_parameters: Optional[dict] = None) -> Predictor:
"""
Create instance of :obj:`tf.contrib.predictor.Predictor` from load_config.
If saved_model was provided and tensorrt_config.use_tensorrt, it will try
to create Predictor using tensorrt. If constructor of tensorrt fails,
saved model predictor will be used.
Parameters
----------
load_config
load config
tensorrt_config
tensorrt config
session_config
session configuration
postprocessors_to_use
which postprocessors to use
model_parameters
model parameters to feed in nested view; only names will be used to get
the placeholders and add to predictor feed_tensors
Returns
-------
predictor
predictor
"""
logger = logging.getLogger(__name__)
if load_config.saved_model:
saved_model_path = load_config.saved_model
if tensorrt_config and tensorrt_config.use_tensorrt:
_predictor = _TensorrtPredictor(saved_model_dir=saved_model_path,
tensorrt_config=tensorrt_config,
config=session_config)
logger.info("Using tensorrt predictor")
else:
_predictor = tf.contrib.predictor.from_saved_model(
saved_model_path, config=session_config)
else:
if tensorrt_config and tensorrt_config.use_tensorrt:
logger.warning(
"currently tensorrt can be used only with saved_model")
# create predictor from meta graph
meta_graph_path = load_config.meta_graph
checkpoint_path = load_config.checkpoint
_predictor = _GraphAndCheckpointPredictor(meta_graph_path,
checkpoint_path,
config=session_config)
if postprocessors_to_use:
# pylint: disable=protected-access
# there is no property setter and so only way to assign it
fetch_tensors_unflatten = nest_utils.unflatten_dict_to_nested(
_predictor._fetch_tensors)
fetch_tensors_unflatten_filtered = {
k: v
for k, v in fetch_tensors_unflatten.items()
if k in postprocessors_to_use
}
fetch_tensors_flatten_filtered = nest_utils.flatten_nested_struct(
fetch_tensors_unflatten_filtered)
_predictor._fetch_tensors = fetch_tensors_flatten_filtered
if model_parameters:
_predictor = _add_model_parameters_to_predictor(
_predictor, model_parameters)
return _predictor
def __getitem__(self, item: Union[int, slice]) -> dict:
buffer_slice_flat = {k: v[item] for k, v in self._buffer_flat.items()}
return nest_utils.unflatten_dict_to_nested(buffer_slice_flat)
def _postprocess_tfrecords(**data):
data_flat = nest_utils.flatten_nested_struct(data)
return nest_utils.unflatten_dict_to_nested(
{k: v + "_pp"
for k, v in data_flat.items()})
def test_parse_tfrecord_example(self, tf_decode_raw,
tf_parse_single_example):
def _get_tfrecords_features():
return {
"data1": "feature_1",
"data2": "feature_data2",
"data3": ["feature_data3_0", "feature_data3_1"],
"data4": {
"sub1": "feature_data4_sub1",
"sub2": "feature_data4_sub2"
}
}
def _get_tfrecords_output_types():
return {"data1": "string_value", "data4/sub1": "float_value"}
def _parse_single_example(example, features):
example_flat = nest_utils.flatten_nested_struct(example, "/")
result = {
k: "-".join([str(example_flat[k]), features[k]])
for k in example_flat
}
return result
def _postprocess_tfrecords(**data):
data_flat = nest_utils.flatten_nested_struct(data)
return nest_utils.unflatten_dict_to_nested(
{k: v + "_pp"
for k, v in data_flat.items()})
tf_decode_raw.side_effect = lambda x, y: x + "_raw"
tf_parse_single_example.side_effect = _parse_single_example
mixin = tf_data_utils.TfRecordsMixin()
mixin.get_tfrecords_features = MagicMock(wraps=_get_tfrecords_features)
mixin.get_tfrecords_output_types = MagicMock(
wraps=_get_tfrecords_output_types)
mixin.postprocess_tfrecords = MagicMock(wraps=_postprocess_tfrecords)
mixin.decode_field = MagicMock(wraps=mixin.decode_field)
result = mixin.parse_tfrecord_example(self.data)
features = _get_tfrecords_features()
features_flat = nest_utils.flatten_nested_struct(features, "/")
data_flat = nest_utils.flatten_nested_struct(self.data, "/")
output_types_flat = nest_utils.flatten_nested_struct(
_get_tfrecords_output_types(), "/")
result_must = nest_utils.unflatten_dict_to_nested(
{
k: "-".join([str(data_flat[k]), features_flat[k]]) +
("_raw_pp" if k in output_types_flat else "_pp")
for k in features_flat
}, "/")
self.assertAllEqual(result_must, result)
mixin.get_tfrecords_features.assert_called_once_with()
mixin.get_tfrecords_output_types.assert_called_once_with()
combine_fn_before_decode = lambda x: "-".join([str(x[0]), x[1]])
decode_values = nest_utils.flatten_nested_struct(
nest_utils.combine_nested([self.data, features],
combine_fun=combine_fn_before_decode),
"/")
decode_field_calls = [
mock_call(each_key, decode_values[each_key],
output_types_flat.get(each_key))
for each_key in decode_values
]
mixin.decode_field.assert_has_calls(decode_field_calls, any_order=True)
data_to_postprocess_must = nest_utils.unflatten_dict_to_nested(
{
k: "-".join([str(data_flat[k]), features_flat[k]]) +
("_raw" if k in output_types_flat else "")
for k in features_flat
}, "/")
mixin.postprocess_tfrecords.assert_called_once_with(
**data_to_postprocess_must)