def test_generate_batches_recurrent(): dataset = DummyDataset(input_dim=2, output_dim=3, num_seqs=20) dataset.init_seq_order(1) batch_gen = dataset.generate_batches(recurrent_net=True, max_seqs=2, batch_size=5) while batch_gen.has_more(): batch_gen.peek_next_n(1) batch_gen.advance(1)
def test_batches_recurrent_1(): dataset = DummyDataset(input_dim=2, output_dim=3, num_seqs=2, seq_len=11) dataset.init_seq_order(1) dataset.chunk_size = 10 dataset.chunk_step = 5 batch_gen = dataset.generate_batches(recurrent_net=True, max_seqs=1, batch_size=20) all_batches = []; " :type: list[Batch] " while batch_gen.has_more(): batch, = batch_gen.peek_next_n(1) assert_is_instance(batch, Batch) print("batch:", batch) print("batch seqs:", batch.seqs) all_batches.append(batch) batch_gen.advance(1) # Each batch will have 1 batch-slice (max_seqs) and up to 10 frames (chunk_size). # For each seq, we get 3 chunks (chunk_step 5 for 11 frames). # Thus, 6 batches. assert_equal(len(all_batches), 6) assert_equal(all_batches[0].start_seq, 0) assert_equal(all_batches[0].end_seq, 1) # exclusive assert_equal(len(all_batches[0].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[0].seqs[0].seq_idx, 0) assert_equal(all_batches[0].seqs[0].seq_start_frame, 0) assert_equal(all_batches[0].seqs[0].seq_end_frame, 10) assert_equal(all_batches[0].seqs[0].frame_length, 10) assert_equal(all_batches[0].seqs[0].batch_slice, 0) assert_equal(all_batches[0].seqs[0].batch_frame_offset, 0) assert_equal(all_batches[1].start_seq, 0) assert_equal(all_batches[1].end_seq, 1) # exclusive assert_equal(len(all_batches[1].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[1].seqs[0].seq_idx, 0) assert_equal(all_batches[1].seqs[0].seq_start_frame, 5) assert_equal(all_batches[1].seqs[0].seq_end_frame, 11) assert_equal(all_batches[1].seqs[0].frame_length, 6) assert_equal(all_batches[1].seqs[0].batch_slice, 0) assert_equal(all_batches[1].seqs[0].batch_frame_offset, 0) assert_equal(all_batches[2].start_seq, 0) assert_equal(all_batches[2].end_seq, 1) # exclusive assert_equal(len(all_batches[2].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[2].seqs[0].seq_idx, 0) assert_equal(all_batches[2].seqs[0].seq_start_frame, 10) assert_equal(all_batches[2].seqs[0].seq_end_frame, 11) assert_equal(all_batches[2].seqs[0].frame_length, 1) assert_equal(all_batches[2].seqs[0].batch_slice, 0) assert_equal(all_batches[2].seqs[0].batch_frame_offset, 0) assert_equal(all_batches[3].start_seq, 1) assert_equal(all_batches[3].end_seq, 2) # exclusive assert_equal(len(all_batches[3].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[3].seqs[0].seq_idx, 1) assert_equal(all_batches[3].seqs[0].seq_start_frame, 0) assert_equal(all_batches[3].seqs[0].seq_end_frame, 10) assert_equal(all_batches[3].seqs[0].frame_length, 10) assert_equal(all_batches[3].seqs[0].batch_slice, 0) assert_equal(all_batches[3].seqs[0].batch_frame_offset, 0)
def test_batches_non_recurrent_1(): dataset = DummyDataset(input_dim=2, output_dim=3, num_seqs=2, seq_len=11) dataset.init_seq_order(1) batch_gen = dataset.generate_batches(recurrent_net=False, max_seqs=2, batch_size=5) all_batches = []; " :type: list[Batch] " while batch_gen.has_more(): batch, = batch_gen.peek_next_n(1) assert_is_instance(batch, Batch) all_batches.append(batch) batch_gen.advance(1) # Each batch will have 5 frames (batch_size), not more, i.e. a single seq. # There are 2 * 11 frames in total, so 5 batches, because we concat the 2 seqs, in the non-recurrent case. assert_equal(len(all_batches), 5) assert_equal(all_batches[0].start_seq, 0) assert_equal(all_batches[0].end_seq, 1) # exclusive assert_equal(len(all_batches[0].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[0].seqs[0].seq_idx, 0) assert_equal(all_batches[0].seqs[0].seq_start_frame, 0) assert_equal(all_batches[0].seqs[0].seq_end_frame, 5) assert_equal(all_batches[0].seqs[0].frame_length, 5) assert_equal(all_batches[0].seqs[0].batch_slice, 0) assert_equal(all_batches[0].seqs[0].batch_frame_offset, 0) assert_equal(all_batches[1].start_seq, 0) assert_equal(all_batches[1].end_seq, 1) # exclusive assert_equal(len(all_batches[1].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[1].seqs[0].seq_idx, 0) assert_equal(all_batches[1].seqs[0].seq_start_frame, 5) assert_equal(all_batches[1].seqs[0].seq_end_frame, 10) assert_equal(all_batches[1].seqs[0].frame_length, 5) assert_equal(all_batches[1].seqs[0].batch_slice, 0) assert_equal(all_batches[1].seqs[0].batch_frame_offset, 0) assert_equal(all_batches[2].start_seq, 0) assert_equal(all_batches[2].end_seq, 2) # exclusive. now both seq 0 and 1 assert_equal(len(all_batches[2].seqs), 2) # two copies, BatchSeqCopyPart assert_equal(all_batches[2].seqs[0].seq_idx, 0) assert_equal(all_batches[2].seqs[0].seq_start_frame, 10) assert_equal(all_batches[2].seqs[0].seq_end_frame, 11) assert_equal(all_batches[2].seqs[0].frame_length, 1) assert_equal(all_batches[2].seqs[0].batch_slice, 0) assert_equal(all_batches[2].seqs[0].batch_frame_offset, 0) assert_equal(all_batches[3].start_seq, 1) assert_equal(all_batches[3].end_seq, 2) # exclusive assert_equal(len(all_batches[3].seqs), 1) # 1 BatchSeqCopyPart assert_equal(all_batches[3].seqs[0].seq_idx, 1) assert_equal(all_batches[3].seqs[0].seq_start_frame, 0) assert_equal(all_batches[3].seqs[0].seq_end_frame, 5) assert_equal(all_batches[3].seqs[0].frame_length, 5) assert_equal(all_batches[3].seqs[0].batch_slice, 0) assert_equal(all_batches[3].seqs[0].batch_frame_offset, 0)
def test_batches_context_window(): context_window = 2 ctx_lr = context_window - 1 ctx_left = ctx_lr // 2 ctx_right = ctx_lr - ctx_left dataset = DummyDataset(input_dim=2, output_dim=3, num_seqs=1, seq_len=11, context_window=context_window) dataset.init_seq_order(1) dataset.chunk_size = 5 dataset.chunk_step = 5 batch_gen = dataset.generate_batches(recurrent_net=True, max_seqs=1, batch_size=20) all_batches = [] # type: list[Batch] while batch_gen.has_more(): batch, = batch_gen.peek_next_n(1) assert_is_instance(batch, Batch) print("batch:", batch) print("batch seqs:", batch.seqs) all_batches.append(batch) batch_gen.advance(1) # Each batch will have 1 batch-slice (max_seqs) and up to 10 frames (chunk_size). # For each seq, we get 3 chunks (chunk_step 5 for 11 frames). # Thus, 3 batches. assert_equal(len(all_batches), 3) b0, b1, b2 = all_batches assert isinstance(b0, Batch) assert isinstance(b1, Batch) assert isinstance(b2, Batch) assert_equal(b0.start_seq, 0) assert_equal(b0.end_seq, 1) # exclusive assert_equal(len(b0.seqs), 1) # 1 BatchSeqCopyPart assert_equal(b0.seqs[0].seq_idx, 0) assert_equal(b0.seqs[0].seq_start_frame["classes"], 0) assert_equal(b0.seqs[0].seq_end_frame["classes"], 5) assert_equal(b0.seqs[0].frame_length["classes"], 5) assert_equal(b0.seqs[0].seq_start_frame["data"], 0 - ctx_left) assert_equal(b0.seqs[0].seq_end_frame["data"], 5 + ctx_right) assert_equal(b0.seqs[0].frame_length["data"], 5 + ctx_lr) assert_equal(b0.seqs[0].batch_slice, 0) assert_equal(b0.seqs[0].batch_frame_offset, 0) assert_equal(b1.start_seq, 0) assert_equal(b1.end_seq, 1) # exclusive assert_equal(len(b1.seqs), 1) # 1 BatchSeqCopyPart assert_equal(b1.seqs[0].seq_idx, 0) assert_equal(b1.seqs[0].seq_start_frame["classes"], 5) assert_equal(b1.seqs[0].seq_end_frame["classes"], 10) assert_equal(b1.seqs[0].frame_length["classes"], 5) assert_equal(b1.seqs[0].seq_start_frame["data"], 5 - ctx_left) assert_equal(b1.seqs[0].seq_end_frame["data"], 10 + ctx_right) assert_equal(b1.seqs[0].frame_length["data"], 5 + ctx_lr) assert_equal(b1.seqs[0].batch_slice, 0) assert_equal(b1.seqs[0].batch_frame_offset, 0) assert_equal(b2.start_seq, 0) assert_equal(b2.end_seq, 1) # exclusive assert_equal(len(b2.seqs), 1) # 1 BatchSeqCopyPart assert_equal(b2.seqs[0].seq_idx, 0) assert_equal(b2.seqs[0].seq_start_frame["classes"], 10) assert_equal(b2.seqs[0].seq_end_frame["classes"], 11) assert_equal(b2.seqs[0].frame_length["classes"], 1) assert_equal(b2.seqs[0].seq_start_frame["data"], 10 - ctx_left) assert_equal(b2.seqs[0].seq_end_frame["data"], 11 + ctx_right) assert_equal(b2.seqs[0].frame_length["data"], 1 + ctx_lr) assert_equal(b2.seqs[0].batch_slice, 0) assert_equal(b2.seqs[0].batch_frame_offset, 0)
def test_batches_context_window(): context_window = 2 ctx_lr = context_window - 1 ctx_left = ctx_lr // 2 ctx_right = ctx_lr - ctx_left dataset = DummyDataset(input_dim=2, output_dim=3, num_seqs=1, seq_len=11, context_window=context_window) dataset.init_seq_order(1) dataset.chunk_size = 5 dataset.chunk_step = 5 batch_gen = dataset.generate_batches(recurrent_net=True, max_seqs=1, batch_size=20) all_batches = [] # type: list[Batch] while batch_gen.has_more(): batch, = batch_gen.peek_next_n(1) assert_is_instance(batch, Batch) print("batch:", batch) print("batch seqs:", batch.seqs) all_batches.append(batch) batch_gen.advance(1) # Each batch will have 1 batch-slice (max_seqs) and up to 10 frames (chunk_size). # For each seq, we get 3 chunks (chunk_step 5 for 11 frames). # Thus, 3 batches. assert_equal(len(all_batches), 3) b0, b1, b2 = all_batches assert isinstance(b0, Batch) assert isinstance(b1, Batch) assert isinstance(b2, Batch) assert_equal(b0.start_seq, 0) assert_equal(b0.end_seq, 1) # exclusive assert_equal(len(b0.seqs), 1) # 1 BatchSeqCopyPart assert_equal(b0.seqs[0].seq_idx, 0) assert_equal(b0.seqs[0].seq_start_frame["classes"], 0) assert_equal(b0.seqs[0].seq_end_frame["classes"], 5) assert_equal(b0.seqs[0].frame_length["classes"], 5) assert_equal(b0.seqs[0].seq_start_frame["data"], 0 - ctx_left) assert_equal(b0.seqs[0].seq_end_frame["data"], 5 + ctx_right) assert_equal(b0.seqs[0].frame_length["data"], 5 + ctx_lr) assert_equal(b0.seqs[0].batch_slice, 0) assert_equal(b0.seqs[0].batch_frame_offset, 0) assert_equal(b1.start_seq, 0) assert_equal(b1.end_seq, 1) # exclusive assert_equal(len(b1.seqs), 1) # 1 BatchSeqCopyPart assert_equal(b1.seqs[0].seq_idx, 0) assert_equal(b1.seqs[0].seq_start_frame["classes"], 5) assert_equal(b1.seqs[0].seq_end_frame["classes"], 10) assert_equal(b1.seqs[0].frame_length["classes"], 5) assert_equal(b1.seqs[0].seq_start_frame["data"], 5 - ctx_left) assert_equal(b1.seqs[0].seq_end_frame["data"], 10 + ctx_right) assert_equal(b1.seqs[0].frame_length["data"], 5 + ctx_lr) assert_equal(b1.seqs[0].batch_slice, 0) assert_equal(b1.seqs[0].batch_frame_offset, 0) assert_equal(b2.start_seq, 0) assert_equal(b2.end_seq, 1) # exclusive assert_equal(len(b2.seqs), 1) # 1 BatchSeqCopyPart assert_equal(b2.seqs[0].seq_idx, 0) assert_equal(b2.seqs[0].seq_start_frame["classes"], 10) assert_equal(b2.seqs[0].seq_end_frame["classes"], 11) assert_equal(b2.seqs[0].frame_length["classes"], 1) assert_equal(b2.seqs[0].seq_start_frame["data"], 10 - ctx_left) assert_equal(b2.seqs[0].seq_end_frame["data"], 11 + ctx_right) assert_equal(b2.seqs[0].frame_length["data"], 1 + ctx_lr) assert_equal(b2.seqs[0].batch_slice, 0) assert_equal(b2.seqs[0].batch_frame_offset, 0)