def testSlideSparse(self): def _sparse(i): return sparse_tensor.SparseTensorValue( indices=[[0]], values=(i * [1]), dense_shape=[1]) iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( sliding.sliding_window_batch( window_size=5, window_shift=3)).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op) num_batches = (10 - 5) // 3 + 1 for i in range(num_batches): actual = sess.run(get_next) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4], dense_shape=[5, 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) self.assertSparseValuesEqual(actual, expected) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next)
def testSlideSparseWithDifferentDenseShapes(self): def _sparse(i): return sparse_tensor.SparseTensorValue( indices=array_ops.expand_dims( math_ops.range(i, dtype=dtypes.int64), 1), values=array_ops.fill([math_ops.to_int32(i)], i), dense_shape=[i]) iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( sliding.sliding_window_batch( window_size=5, window_shift=3)).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op) num_batches = (10 - 5) // 3 + 1 for i in range(num_batches): actual = sess.run(get_next) expected_indices = [] expected_values = [] for j in range(5): for k in range(i * 3 + j): expected_indices.append([j, k]) expected_values.append(i * 3 + j) expected = sparse_tensor.SparseTensorValue( indices=expected_indices, values=expected_values, dense_shape=[5, i * 3 + 5 - 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) self.assertSparseValuesEqual(actual, expected) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next)
def testSlideDatasetInvalid(self, count, window_size, window_shift, window_stride): count_t = array_ops.placeholder(dtypes.int64, shape=[]) window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) iterator = ( dataset_ops.Dataset.range(10).map(lambda x: x).repeat(count_t).apply( sliding.sliding_window_batch( window_size=window_size_t, window_shift=window_shift_t, window_stride=window_stride_t)).make_initializable_iterator()) init_op = iterator.initializer with self.cached_session() as sess: with self.assertRaises(errors.InvalidArgumentError): sess.run( init_op, feed_dict={ count_t: count, window_size_t: window_size, window_shift_t: window_shift, window_stride_t: window_stride })
def testNestedSlideSparse(self): def _sparse(i): return sparse_tensor.SparseTensorValue( indices=[[0]], values=(i * [1]), dense_shape=[1]) iterator = ( dataset_ops.Dataset.range(10).map(_sparse).apply( sliding.sliding_window_batch(window_size=4, window_shift=2)).apply( sliding.sliding_window_batch(window_size=3, window_shift=1)) .make_initializable_iterator()) init_op = iterator.initializer get_next = iterator.get_next() with self.cached_session() as sess: sess.run(init_op) # Slide: 1st batch. actual = sess.run(get_next) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]], values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7], dense_shape=[3, 4, 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) self.assertSparseValuesEqual(actual, expected) # Slide: 2nd batch. actual = sess.run(get_next) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]], values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9], dense_shape=[3, 4, 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) self.assertSparseValuesEqual(actual, expected) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next)
def testSlideDataset(self, count, window_size, window_shift, window_stride): """Tests a dataset that slides a window its input elements.""" components = (np.arange(7), np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], np.array(37.0) * np.arange(7)) count_t = array_ops.placeholder(dtypes.int64, shape=[]) window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> # RepeatDataset(count) -> # _SlideDataset(window_size, window_shift, window_stride). iterator = ( dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) .repeat(count).apply( sliding.sliding_window_batch( window_size=window_size_t, window_shift=window_shift_t, window_stride=window_stride_t)).make_initializable_iterator()) init_op = iterator.initializer get_next = iterator.get_next() self.assertEqual([[None] + list(c.shape[1:]) for c in components], [t.shape.as_list() for t in get_next]) with self.cached_session() as sess: sess.run( init_op, feed_dict={ count_t: count, window_size_t: window_size, window_shift_t: window_shift, window_stride_t: window_stride }) num_batches = (count * 7 - ( (window_size - 1) * window_stride + 1)) // window_shift + 1 for i in range(num_batches): result = sess.run(get_next) for component, result_component in zip(components, result): for j in range(window_size): self.assertAllEqual( component[(i * window_shift + j * window_stride) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next)
def testSlideShapeError(self): def generator(): yield [1.0, 2.0, 3.0] yield [4.0, 5.0, 6.0] yield [7.0, 8.0, 9.0, 10.0] iterator = dataset_ops.make_initializable_iterator( dataset_ops.Dataset.from_generator( generator, dtypes.float32, output_shapes=[None]).apply( sliding.sliding_window_batch(window_size=3, window_shift=1))) next_element = iterator.get_next() with self.cached_session() as sess: sess.run(iterator.initializer) with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Cannot batch tensors with different shapes in component 0. " r"First element had shape \[3\] and element 2 had shape \[4\]."): sess.run(next_element)
def testSlideShapeError(self): def generator(): yield [1.0, 2.0, 3.0] yield [4.0, 5.0, 6.0] yield [7.0, 8.0, 9.0, 10.0] iterator = (dataset_ops.Dataset.from_generator( generator, dtypes.float32, output_shapes=[None]).apply( sliding.sliding_window_batch( window_size=3, window_shift=1)).make_initializable_iterator()) next_element = iterator.get_next() with self.cached_session() as sess: sess.run(iterator.initializer) with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Cannot batch tensors with different shapes in component 0. " r"First element had shape \[3\] and element 2 had shape \[4\]." ): sess.run(next_element)
def testSlideDatasetInvalid(self, count, window_size, window_shift, window_stride): count_t = array_ops.placeholder(dtypes.int64, shape=[]) window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) iterator = (dataset_ops.Dataset.range(10).map(lambda x: x).repeat( count_t).apply( sliding.sliding_window_batch(window_size=window_size_t, window_shift=window_shift_t, window_stride=window_stride_t)). make_initializable_iterator()) init_op = iterator.initializer with self.cached_session() as sess: with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={ count_t: count, window_size_t: window_size, window_shift_t: window_shift, window_stride_t: window_stride })
def testSlideSparse(self): def _sparse(i): return sparse_tensor.SparseTensorValue(indices=[[0]], values=(i * [1]), dense_shape=[1]) iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( sliding.sliding_window_batch(5, 3)).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() with self.test_session() as sess: sess.run(init_op) num_batches = (10 - 5) // 3 + 1 for i in range(num_batches): actual = sess.run(get_next) expected = sparse_tensor.SparseTensorValue( indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]], values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4], dense_shape=[5, 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) self.assertSparseValuesEqual(actual, expected) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next)
def testSlideDatasetValueError(self): with self.assertRaises(ValueError): dataset_ops.Dataset.range(10).map(lambda x: x).apply( sliding.sliding_window_batch( window_size=1, stride=1, window_shift=1, window_stride=1))
def testSlideDataset(self): """Test an dataset that maps a TF function across its input elements.""" components = (np.arange(7), np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], np.array(37.0) * np.arange(7)) count = array_ops.placeholder(dtypes.int64, shape=[]) window_size = array_ops.placeholder(dtypes.int64, shape=[]) stride = array_ops.placeholder(dtypes.int64, shape=[]) def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> # RepeatDataset(count) -> _SlideDataset(window_size, stride). iterator = (dataset_ops.Dataset.from_tensor_slices(components).map( _map_fn).repeat(count).apply( sliding.sliding_window_batch( window_size, stride)).make_initializable_iterator()) init_op = iterator.initializer get_next = iterator.get_next() self.assertEqual([[None] + list(c.shape[1:]) for c in components], [t.shape.as_list() for t in get_next]) with self.test_session() as sess: # Slide over a finite input, where the window_size divides the # total number of elements. sess.run(init_op, feed_dict={ count: 20, window_size: 14, stride: 7 }) # Same formula with convolution layer. num_batches = (20 * 7 - 14) // 7 + 1 for i in range(num_batches): result = sess.run(get_next) for component, result_component in zip(components, result): for j in range(14): self.assertAllEqual(component[(i * 7 + j) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Slide over a finite input, where the window_size does not # divide the total number of elements. sess.run(init_op, feed_dict={ count: 20, window_size: 17, stride: 9 }) num_batches = (20 * 7 - 17) // 9 + 1 for i in range(num_batches): result = sess.run(get_next) for component, result_component in zip(components, result): for j in range(17): self.assertAllEqual(component[(i * 9 + j) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Slide over a finite input, which is less than window_size, # should fail straight away. sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 4}) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 8}) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Slide over an empty input should fail straight away. sess.run(init_op, feed_dict={count: 0, window_size: 8, stride: 4}) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Empty window_size should be an initialization time error. with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={ count: 14, window_size: 0, stride: 0 }) # Invalid stride should be an initialization time error. with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={ count: 14, window_size: 3, stride: 0 }) with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={ count: 14, window_size: 3, stride: 3 }) with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={ count: 14, window_size: 3, stride: 5 })
def testSlideDataset(self): """Test an dataset that maps a TF function across its input elements.""" components = (np.arange(7), np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], np.array(37.0) * np.arange(7)) count = array_ops.placeholder(dtypes.int64, shape=[]) window_size = array_ops.placeholder(dtypes.int64, shape=[]) stride = array_ops.placeholder(dtypes.int64, shape=[]) def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> # RepeatDataset(count) -> _SlideDataset(window_size, stride). iterator = (dataset_ops.Dataset.from_tensor_slices(components) .map(_map_fn) .repeat(count) .apply(sliding.sliding_window_batch(window_size, stride)) .make_initializable_iterator()) init_op = iterator.initializer get_next = iterator.get_next() self.assertEqual([[None] + list(c.shape[1:]) for c in components], [t.shape.as_list() for t in get_next]) with self.test_session() as sess: # Slide over a finite input, where the window_size divides the # total number of elements. sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 7}) # Same formula with convolution layer. num_batches = (20 * 7 - 14) // 7 + 1 for i in range(num_batches): result = sess.run(get_next) for component, result_component in zip(components, result): for j in range(14): self.assertAllEqual(component[(i*7 + j) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Slide over a finite input, where the window_size does not # divide the total number of elements. sess.run(init_op, feed_dict={count: 20, window_size: 17, stride: 9}) num_batches = (20 * 7 - 17) // 9 + 1 for i in range(num_batches): result = sess.run(get_next) for component, result_component in zip(components, result): for j in range(17): self.assertAllEqual(component[(i*9 + j) % 7]**2, result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Slide over a finite input, which is less than window_size, # should fail straight away. sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 4}) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 8}) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Slide over an empty input should fail straight away. sess.run(init_op, feed_dict={count: 0, window_size: 8, stride: 4}) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) # Empty window_size should be an initialization time error. with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={count: 14, window_size: 0, stride: 0}) # Invalid stride should be an initialization time error. with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 0}) with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 3}) with self.assertRaises(errors.InvalidArgumentError): sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 5})
def model(learning_rate,num_epochs,mini_size,pt_out,break_t,fil_conv,kernel_ls,decode_l, pera_1,pera_2,imp_skip,batch): tf.summary.scalar('learning_rate',learning_rate) tf.summary.scalar('batch_size',mini_size) tf.summary.scalar('epoch_num',num_epochs) tf.summary.scalar('out_step',pt_out) tf.summary.scalar('training_break',break_t) tf.summary.scalar('conv_filter_numbers',fil_conv) tf.summary.scalar('kernel_sizes_lstm',kernel_ls) tf.summary.scalar('number_of_prediction',decode_l) tf.summary.scalar('batch',batch) filenames = tf.placeholder(tf.string) dataset = tf.data.TFRecordDataset(filenames) dataset = dataset.map(_parse_function) window = mini_size #stride = 1 dataset = dataset.apply(sliding.sliding_window_batch(window,stride=6)) dataset = dataset.batch(batch,drop_remainder=True) dataset = dataset.shuffle(500) dataset = dataset.repeat(num_epochs) #dataset = dataset.shuffle(500) #iterator = tf.data.Iterator.from_structure(dataset.output_types,dataset.output_shapes) iterator = dataset.make_initializable_iterator(shared_name="iter") pix_gt = iterator.get_next() #print(pix_gt.get_shape().as_list()) #print(tf.shape(pix_gt)) spli0, spli1, spli2, spli3, spli4, spli5 = tf.split(pix_gt,num_or_size_splits=batch,axis=0) spli0 = tf.reshape(spli0,[mini_size,256,256,1]) spli1 = tf.reshape(spli1,[mini_size,256,256,1]) spli2 = tf.reshape(spli2,[mini_size,256,256,1]) spli3 = tf.reshape(spli3,[mini_size,256,256,1]) spli4 = tf.reshape(spli4,[mini_size,256,256,1]) spli5 = tf.reshape(spli5,[mini_size,256,256,1]) pix_gt1 = tf.stack([spli0, spli1, spli2, spli3, spli4, spli5], axis=1) split0, split1, split2, split3, split4, split5 = tf.split(pix_gt1, num_or_size_splits=mini_size, axis=0) pix_0 = tf.reshape(split0,[batch,256,256,1]) pix_1 = tf.reshape(split1,[batch,256,256,1]) pix_2 = tf.reshape(split2,[batch,256,256,1]) pix_3 = tf.reshape(split3,[batch,256,256,1]) pix_4 = tf.reshape(split4,[batch,256,256,1]) pix_5 = tf.reshape(split5,[batch,256,256,1]) tf.summary.image("input_1",pix_0,3) tf.summary.image("input_2",pix_1,3) tf.summary.image("input_3",pix_2,3) tf.summary.image("input_4",pix_3,3) tf.summary.image("input_5",pix_4,3) tf.summary.image("input_6",pix_5,3) out_1,out_2,out_3,out_4,two_pix,two_pixel_1,two_pixel_2,two_pixel_3,two_pixel_4,two_pixel_5 = encoder_block(pix_0,pix_1,pix_2,pix_3) #out_5 = lstm_block(out_1,out_2,out_3,out_4,) out_5,out_6 = lstm_block(out_1,out_2,out_3,out_4,pix_4) out_pre,out_pre1 = decoder_block(out_5,out_6,two_pixel_5,two_pixel_4,two_pixel_3,two_pixel_2,two_pixel_1,two_pix, skip_try="oka") tf.summary.image("prediction1",out_pre,3) tf.summary.image("prediction2",out_pre1,3) #loss = cost(pixel_pre = out_pre , pixel_gt = pix_4) loss = cost(out_pre,out_pre1,pix_4,pix_5,pera_1,pera_2) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,name="adam").minimize(loss) merge_sum = tf.summary.merge_all() file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph()) saver = tf.train.Saver() init = tf.global_variables_initializer() sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) sess.run(init) #saver.restore(sess,('/media/antor/Files/main_projects/gitlab/unet_check/tf_models/run-20181104123103/my_model.ckpt')) sess.run(iterator.initializer,feed_dict={filenames:"/media/antor/Files/ML/tfrecord/sir_demo/train.tfrecords"}) mini_cost = 0.0 counter = 1 coun = 1 epoch_cost = 0.0 epoch = 0 ''' while True: try: if coun in range(imp_skip,31): dump_loss = sess.run(loss) #print(coun) if coun==30: coun=0 coun+=1 else: _ , temp_cost = sess.run([optimizer,loss]) mini_cost += temp_cost/pt_out epoch_cost += temp_cost/8200 if counter%pt_out==0: print("mini batch cost of batch " + str(counter) + " is : " + str(mini_cost)) mini_cost =0.0 if counter%100 == 0: s = sess.run(merge_sum) file_writer.add_summary(s,counter) #if counter*mini_size>=break_t: # break if counter%10000==0: print("cost after epoch " + str(epoch) + ": " + str(epoch_cost)) saver.save(sess,logdir_m+"my_model.ckpt") epoch_cost =0.0 epoch+=1 counter+=1 #print(coun) coun+=1 except tf.errors.OutOfRangeError: print(counter) saver.save(sess,logdir_m+"my_model.ckpt") break ''' while True: try: ''' #if coun in range(imp_skip,31): #if coun==5: dump_loss = sess.run(loss) #print(coun) #if coun==30: coun=0 coun+=1 else: ''' _ , temp_cost = sess.run([optimizer,loss]) mini_cost += temp_cost/pt_out epoch_cost += temp_cost/288 # if counter%pt_out==0: # print("mini batch cost of batch " + str(counter) + " is : " + str(mini_cost)) # mini_cost =0.0 if counter%288== 0: s = sess.run(merge_sum) file_writer.add_summary(s,counter) #if counter*mini_size>=break_t: # break if counter%288==0: print("cost after epoch " + str(epoch) + ": " + str(epoch_cost)) #saver.save(sess,logdir_m+"my_model.ckpt") epoch_cost =0.0 epoch+=1 counter+=1 #print(coun) coun+=1 except tf.errors.OutOfRangeError: #print(counter) #saver.save(sess,logdir_m+"my_model.ckpt") break sess.close()