def example_protos_from_path(cns_path, num_examples=10, start_index=0, parse_examples=True): """Returns a number of tf.train.Examples from the CNS path. Args: cns_path: A string CNS path. num_examples: The maximum number of examples to return from the path. start_index: The index of the first example to return. parse_examples: If true then parses the serialized proto from the path into proto objects. Defaults to True. Returns: A list of Example protos or serialized proto strings at the CNS path. Raises: InvalidUserInputError: If examples cannot be procured from cns_path. """ def append_examples_from_iterable(iterable, examples): for i, value in enumerate(iterable): if i >= start_index: examples.append( tf.train.Example.FromString(value ) if parse_examples else value) if len(examples) >= num_examples: return filenames = filepath_to_filepath_list(cns_path) examples = [] compression_types = [ tf.python_io.TFRecordCompressionType.NONE, tf.python_io.TFRecordCompressionType.GZIP, tf.python_io.TFRecordCompressionType.ZLIB, ] current_compression_idx = 0 current_file_index = 0 while (current_file_index < len(filenames) and current_compression_idx < len(compression_types)): try: record_iterator = tf.python_io.tf_record_iterator( path=filenames[current_file_index], options=tf.python_io.TFRecordOptions( compression_types[current_compression_idx])) append_examples_from_iterable(record_iterator, examples) current_file_index += 1 if len(examples) >= num_examples: break except tf.errors.DataLossError: current_compression_idx += 1 except (IOError, tf.errors.NotFoundError) as e: raise common_utils.InvalidUserInputError(e) if examples: return examples else: raise common_utils.InvalidUserInputError( 'No tf.train.Examples found at ' + cns_path + '. Valid formats are TFRecord files.')
def convert_pattern_to_indices(pattern): """Converts a printer-page-style pattern and returns a list of indices. Args: pattern: A printer-page-style pattern with only numeric characters, commas, dashes, and optionally spaces. For example, a pattern of '0,2,4-6' would yield [0, 2, 4, 5, 6]. Returns: A list of indices represented by the pattern. """ pieces = [token.strip() for token in pattern.split(',')] indices = [] for piece in pieces: try: if '-' in piece: lower, upper = [ int(x.strip()) for x in piece.split('-') ] indices.extend(range(lower, upper + 1)) else: indices.append(int(piece.strip())) except ValueError as e: raise common_utils.InvalidUserInputError(e) return sorted(indices)
def mutant_charts_for_feature(example_protos, feature_name, serving_bundles, viz_params): """Returns JSON formatted for rendering all charts for a feature. Args: example_proto: The example protos to mutate. feature_name: The string feature name to mutate. serving_bundles: One `ServingBundle` object per model, that contains the information to make the serving request. viz_params: A `VizParams` object that contains the UI state of the request. Raises: InvalidUserInputError if `viz_params.feature_index_pattern` requests out of range indices for `feature_name` within `example_proto`. Returns: A JSON-able dict for rendering a single mutant chart. parsed in `tf-inference-dashboard.html`. { 'chartType': 'numeric', # oneof('numeric', 'categorical') 'data': [A list of data] # parseable by vz-line-chart or vz-bar-chart } """ def chart_for_index(index_to_mutate): mutant_features, mutant_examples = make_mutant_tuples( example_protos, original_feature, index_to_mutate, viz_params) charts = [] for serving_bundle in serving_bundles: (inference_result_proto, _) = run_inference( mutant_examples, serving_bundle) charts.append(make_json_formatted_for_single_chart( mutant_features, inference_result_proto, index_to_mutate)) return charts try: original_feature = parse_original_feature_from_example( example_protos[0], feature_name) except ValueError as e: return { 'chartType': 'categorical', 'data': [] } indices_to_mutate = viz_params.feature_indices or range( original_feature.length) chart_type = ('categorical' if original_feature.feature_type == 'bytes_list' else 'numeric') try: return { 'chartType': chart_type, 'data': [ chart_for_index(index_to_mutate) for index_to_mutate in indices_to_mutate ] } except IndexError as e: raise common_utils.InvalidUserInputError(e)
def example_protos_from_path(cns_path, num_examples=10, start_index=0, parse_examples=True): """Returns a number of tf.train.Examples from the CNS path. Args: cns_path: A string CNS path. num_examples: The maximum number of examples to return from the path. start_index: The index of the first example to return. parse_examples: If true then parses the serialized proto from the path into proto objects. Defaults to True. Returns: A list of Example protos or serialized proto strings at the CNS path. Raises: InvalidUserInputError: If examples cannot be procured from cns_path. """ def append_examples_from_iterable(iterable, examples): for i, value in enumerate(iterable): if i >= start_index: examples.append( tf.train.Example.FromString(value) if parse_examples else value) if len(examples) >= num_examples: return filenames = filepath_to_filepath_list(cns_path) examples = [] try: for filename in filenames: record_iterator = tf.python_io.tf_record_iterator(path=filename) append_examples_from_iterable(record_iterator, examples) if len(examples) >= num_examples: break except (IOError, tf.errors.NotFoundError) as e: raise common_utils.InvalidUserInputError(e) if examples: return examples else: raise common_utils.InvalidUserInputError( 'No tf.train.Examples found at ' + cns_path + '. Valid formats are SSTable and RecordIO.')
def _parse_request_arguments(self, request): """Parses comma separated request arguments Args: request: A request that should contain 'inference_address', 'model_name', 'model_version', 'model_signature'. Returns: A tuple of lists for model parameters """ inference_addresses = request.args.get('inference_address').split(',') model_names = request.args.get('model_name').split(',') model_versions = request.args.get('model_version').split(',') model_signatures = request.args.get('model_signature').split(',') if len(model_names) != len(inference_addresses): raise common_utils.InvalidUserInputError('Every model should have a ' + 'name and address.') return inference_addresses, model_names, model_versions, model_signatures
def example_protos_from_path(path, num_examples=10, start_index=0, parse_examples=True, sampling_odds=1, example_class=tf.train.Example): """Returns a number of examples from the provided path. Args: path: A string path to the examples. num_examples: The maximum number of examples to return from the path. parse_examples: If true then parses the serialized proto from the path into proto objects. Defaults to True. sampling_odds: Odds of loading an example, used for sampling. When >= 1 (the default), then all examples are loaded. example_class: tf.train.Example or tf.train.SequenceExample class to load. Defaults to tf.train.Example. Returns: A list of Example protos or serialized proto strings at the path. Raises: InvalidUserInputError: If examples cannot be procured from the path. """ def append_examples_from_iterable(iterable, examples): for value in iterable: if sampling_odds >= 1 or random.random() < sampling_odds: examples.append( example_class.FromString(value ) if parse_examples else value) if len(examples) >= num_examples: return examples = [] if path.endswith('.csv'): def are_floats(values): for value in values: try: float(value) except ValueError: return False return True csv.register_dialect('CsvDialect', skipinitialspace=True) rows = csv.DictReader(open(path), dialect='CsvDialect') for row in rows: if sampling_odds < 1 and random.random() > sampling_odds: continue example = tf.train.Example() for col in row.keys(): # Parse out individual values from vertical-bar-delimited lists values = [val.strip() for val in row[col].split('|')] if are_floats(values): example.features.feature[col].float_list.value.extend( [float(val) for val in values]) else: example.features.feature[col].bytes_list.value.extend( [val.encode('utf-8') for val in values]) examples.append( example if parse_examples else example.SerializeToString()) if len(examples) >= num_examples: break return examples filenames = filepath_to_filepath_list(path) compression_types = [ '', # no compression (distinct from `None`!) 'GZIP', 'ZLIB', ] current_compression_idx = 0 current_file_index = 0 while (current_file_index < len(filenames) and current_compression_idx < len(compression_types)): try: record_iterator = tf.compat.v1.python_io.tf_record_iterator( path=filenames[current_file_index], options=tf.io.TFRecordOptions( compression_types[current_compression_idx])) append_examples_from_iterable(record_iterator, examples) current_file_index += 1 if len(examples) >= num_examples: break except tf.errors.DataLossError: current_compression_idx += 1 except (IOError, tf.errors.NotFoundError) as e: raise common_utils.InvalidUserInputError(e) if examples: return examples else: raise common_utils.InvalidUserInputError( 'No examples found at ' + path + '. Valid formats are TFRecord files.')
def to_int(x): try: return int(x) except (ValueError, TypeError) as e: raise common_utils.InvalidUserInputError(e)
def example_protos_from_path(path, num_examples=10, start_index=0, parse_examples=True, sampling_odds=1, example_class=tf.train.Example): """Returns a number of examples from the provided path. Args: path: A string path to the examples. num_examples: The maximum number of examples to return from the path. parse_examples: If true then parses the serialized proto from the path into proto objects. Defaults to True. sampling_odds: Odds of loading an example, used for sampling. When >= 1 (the default), then all examples are loaded. example_class: tf.train.Example or tf.train.SequenceExample class to load. Defaults to tf.train.Example. Returns: A list of Example protos or serialized proto strings at the path. Raises: InvalidUserInputError: If examples cannot be procured from the path. """ def append_examples_from_iterable(iterable, examples): for value in iterable: if sampling_odds >= 1 or random.random() < sampling_odds: examples.append( example_class.FromString(value ) if parse_examples else value) if len(examples) >= num_examples: return filenames = filepath_to_filepath_list(path) examples = [] compression_types = [ tf.python_io.TFRecordCompressionType.NONE, tf.python_io.TFRecordCompressionType.GZIP, tf.python_io.TFRecordCompressionType.ZLIB, ] current_compression_idx = 0 current_file_index = 0 while (current_file_index < len(filenames) and current_compression_idx < len(compression_types)): try: record_iterator = tf.python_io.tf_record_iterator( path=filenames[current_file_index], options=tf.python_io.TFRecordOptions( compression_types[current_compression_idx])) append_examples_from_iterable(record_iterator, examples) current_file_index += 1 if len(examples) >= num_examples: break except tf.errors.DataLossError: current_compression_idx += 1 except (IOError, tf.errors.NotFoundError) as e: raise common_utils.InvalidUserInputError(e) if examples: return examples else: raise common_utils.InvalidUserInputError( 'No examples found at ' + path + '. Valid formats are TFRecord files.')