def testIgnoreErrWithUnquotedQuotes(self): record_defaults = [['']] * 3 inputs = [['1,2"3,4', 'a,b,c"d', '9,8"7,6,5', 'e,f,g']] filenames = self._setup_files(inputs) dataset = readers.CsvDataset(filenames, record_defaults=record_defaults) dataset = dataset.apply(error_ops.ignore_errors()) self.assertDatasetProduces(dataset, [(b'e', b'f', b'g')])
def testReadFileIgnoreError(self): def write_string_to_file(value, filename): with open(filename, "w") as f: f.write(value) filenames = [ os.path.join(self.get_temp_dir(), "file_%d.txt" % i) for i in range(5) ] for filename in filenames: write_string_to_file(filename, filename) dataset = ( dataset_ops.Dataset.from_tensor_slices(filenames).map( io_ops.read_file, num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) get_next = self.getNext(dataset) # All of the files are present. for filename in filenames: self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next()) # Delete one of the files. os.remove(filenames[0]) # Attempting to read filenames[0] will fail, but ignore_errors() # will catch the error. get_next = self.getNext(dataset) for filename in filenames[1:]: self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def testReadFileIgnoreError(self): def write_string_to_file(value, filename): with open(filename, "w") as f: f.write(value) filenames = [ os.path.join(self.get_temp_dir(), "file_%d.txt" % i) for i in range(5) ] for filename in filenames: write_string_to_file(filename, filename) dataset = (dataset_ops.Dataset.from_tensor_slices(filenames).map( io_ops.read_file, num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) get_next = self.getNext(dataset) # All of the files are present. for filename in filenames: self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next()) # Delete one of the files. os.remove(filenames[0]) # Attempting to read filenames[0] will fail, but ignore_errors() # will catch the error. get_next = self.getNext(dataset) for filename in filenames[1:]: self.assertEqual(compat.as_bytes(filename), self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def testCsvDataset_ignoreErrWithUnquotedQuotes(self): record_defaults = [['']] * 3 inputs = [['1,2"3,4', 'a,b,c"d', '9,8"7,6,5', 'e,f,g']] filenames = self._setup_files(inputs) dataset = readers.CsvDataset(filenames, record_defaults=record_defaults) dataset = dataset.apply(error_ops.ignore_errors()) self._verify_output_or_err(dataset, [['e', 'f', 'g']])
def ignore_errors(): """Creates a `Dataset` from another `Dataset` and silently ignores any errors. Use this transformation to produce a dataset that contains the same elements as the input, but silently drops any elements that caused an error. For example: ```python dataset = tf.data.Dataset.from_tensor_slices([1., 2., 0., 4.]) # Computing `tf.debugging.check_numerics(1. / 0.)` will raise an InvalidArgumentError. dataset = dataset.map(lambda x: tf.debugging.check_numerics(1. / x, "error")) # Using `ignore_errors()` will drop the element that causes an error. dataset = dataset.apply(tf.data.experimental.ignore_errors()) # ==> { 1., 0.5, 0.2 } ``` Returns: A `Dataset` transformation function, which can be passed to `tf.data.Dataset.apply`. """ return error_ops.ignore_errors()
def testCsvDataset_ignoreErrWithUnquotedQuotes(self): record_defaults = [['']] * 3 inputs = [['1,2"3,4', 'a,b,c"d', '9,8"7,6,5', 'e,f,g']] filenames = self._setup_files(inputs) dataset = readers.CsvDataset(filenames, record_defaults=record_defaults) dataset = dataset.apply(error_ops.ignore_errors()) self._verify_output_or_err(dataset, [['e', 'f', 'g']])
def testCsvDataset_ignoreErrWithUnescapedQuotes(self): record_defaults = [['']] * 3 inputs = [['1,"2"3",4', '1,"2"3",4",5,5', 'a,b,"c"d"', 'e,f,g']] filenames = self._setup_files(inputs) dataset = readers.CsvDataset(filenames, record_defaults=record_defaults) dataset = dataset.apply(error_ops.ignore_errors()) self.assertDatasetProduces(dataset, [(b'e', b'f', b'g')])
def _build_ds(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) dataset = dataset_ops.Dataset.from_tensor_slices(components) dataset = dataset.map(lambda x: array_ops.check_numerics(x, "message")) dataset = dataset.apply(error_ops.ignore_errors()) options = options_lib.Options() options.experimental_external_state_policy = ( options_lib.ExternalStatePolicy.IGNORE) return dataset.with_options(options)
def testZipIgnoreError(self): a = dataset_ops.Dataset.from_tensor_slices([1., 2., 0., 4.]) b = a.map(lambda x: array_ops.check_numerics(1. / x, "error")) dataset = dataset_ops.Dataset.zip((b, a)).apply(error_ops.ignore_errors()) get_next = self.getNext(dataset) for x in [1., 2., 4.]: self.assertEqual((1. / x, x), self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def filename_to_dataset(filename): dataset = CsvDataset(filename, record_defaults=column_defaults, field_delim=field_delim, use_quote_delim=use_quote_delim, na_value=na_value, select_cols=select_columns, header=header, compression_type=compression_type) if ignore_errors: dataset = dataset.apply(error_ops.ignore_errors()) return dataset
def testParallelMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) dataset = (dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) get_next = self.getNext(dataset) for x in [1., 2., 3., 5.]: self.assertEqual(x, self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def testParallelMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) dataset = ( dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) get_next = self.getNext(dataset) for x in [1., 2., 3., 5.]: self.assertEqual(x, self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def testIgnoreError_withLogWarning(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) dataset = (dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message")).apply( error_ops.ignore_errors(log_warning=True))) get_next = self.getNext(dataset) for x in [1., 2., 3.]: self.assertEqual(x, self.evaluate(get_next())) with self.captureWritesToStream(sys.stderr) as logged: self.assertEqual(5., self.evaluate(get_next())) expected = "Tensor had NaN values" self.assertIn((expected), logged.contents()) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def filename_to_dataset(filename): dataset = CsvDataset( filename, record_defaults=column_defaults, field_delim=field_delim, use_quote_delim=use_quote_delim, na_value=na_value, select_cols=select_columns, header=header, compression_type=compression_type ) if ignore_errors: dataset = dataset.apply(error_ops.ignore_errors()) return dataset
def testParallelMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) dataset = (dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: self.evaluate(init_op) for x in [1., 2., 3., 5.]: self.assertEqual(x, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next)
def testParallelMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) dataset = ( dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message"), num_parallel_calls=2).prefetch(2).apply(error_ops.ignore_errors())) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: self.evaluate(init_op) for x in [1., 2., 3., 5.]: self.assertEqual(x, self.evaluate(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next)
def testTFRecordDatasetIgnoreError(self): filenames = [] for i in range(5): fn = os.path.join(self.get_temp_dir(), "tf_record.%d.txt" % i) filenames.append(fn) writer = python_io.TFRecordWriter(fn) for _ in range(10): writer.write(b"record") writer.close() # Append corrupted data with open(fn, "a") as f: f.write("corrupted data") dataset = readers.TFRecordDataset(filenames).apply( error_ops.ignore_errors()) get_next = self.getNext(dataset) # All of the files are present. for _ in filenames: for _ in range(10): self.assertEqual(b"record", self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def testTFRecordDatasetIgnoreError(self): filenames = [] for i in range(5): fn = os.path.join(self.get_temp_dir(), "tf_record.%d.txt" % i) filenames.append(fn) writer = python_io.TFRecordWriter(fn) for j in range(10): writer.write(b"record") writer.close() # Append corrupted data with open(fn, "a") as f: f.write("corrupted data") dataset = readers.TFRecordDataset(filenames).apply( error_ops.ignore_errors()) get_next = self.getNext(dataset) # All of the files are present. for filename in filenames: for j in range(10): self.assertEqual(b"record", self.evaluate(get_next())) with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next())
def ignore_errors(): """Creates a `Dataset` from another `Dataset` and silently ignores any errors. Use this transformation to produce a dataset that contains the same elements as the input, but silently drops any elements that caused an error. For example: ```python dataset = tf.data.Dataset.from_tensor_slices([1., 2., 0., 4.]) # Computing `tf.check_numerics(1. / 0.)` will raise an InvalidArgumentError. dataset = dataset.map(lambda x: tf.check_numerics(1. / x, "error")) # Using `ignore_errors()` will drop the element that causes an error. dataset = dataset.apply(tf.contrib.data.ignore_errors()) # ==> { 1., 0.5, 0.2 } ``` Returns: A `Dataset` transformation function, which can be passed to `tf.data.Dataset.apply`. """ return error_ops.ignore_errors()
def _build_ds(self): return dataset_ops.Dataset.range(5).map( array_ops.ones).map(lambda x: array_ops.gather(x, [0])).apply( error_ops.ignore_errors())
def _build_ds(self, components): return dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message")).apply( error_ops.ignore_errors())
def testCardinality(self): ds = dataset_ops.Dataset.range(10).apply(error_ops.ignore_errors()) self.assertEqual(self.evaluate(ds.cardinality()), dataset_ops.UNKNOWN)
def _build_ds(self, components): return dataset_ops.Dataset.from_tensor_slices(components).map( lambda x: array_ops.check_numerics(x, "message")).apply( error_ops.ignore_errors())