def my_p_converter_wordvec(batch, device=None): Xp1 = [to_device(device, np.asarray(b[0], np.int32)) for b in batch] Xp2 = [to_device(device, np.asarray(b[1], np.int32)) for b in batch] Y = [b[-1] for b in batch] Y = to_device(device, np.asarray(Y, np.int32)[:, None]) return Xp1, Xp2, Y
def converter(batch, device, max_caption_length=None): """Optional preprocessing of the batch before forward pass.""" pad = max_caption_length is not None imgs = [] captions = [] for img, caption in batch: # Preproess the caption by either fixing the length by padding (LSTM) # or by simply wrapping each caption in an ndarray (NStepLSTM) if pad: arr = np.full(max_caption_length, _ignore, dtype=np.int32) # Clip to max length if necessary arr[:len(caption)] = caption[:max_caption_length] caption = arr else: caption = to_device(device, np.asarray(caption, dtype=np.int32)) imgs.append(img) captions.append(caption) if pad: captions = to_device(device, np.stack(captions)) imgs = to_device(device, np.stack(imgs)) return imgs, captions
def concat_examples(batch, device=None, padding=None, indices_concat=None, indices_to_device=None): if len(batch) == 0: raise ValueError('batch is empty') first_elem = batch[0] elem_size = len(first_elem) if indices_concat is None: indices_concat = range(elem_size) if indices_to_device is None: indices_to_device = range(elem_size) result = [] if not isinstance(padding, tuple): padding = [padding] * elem_size for i in six.moves.range(elem_size): res = [example[i] for example in batch] if i in indices_concat: res = _concat_arrays(res, padding[i]) if i in indices_to_device: if i in indices_concat: res = to_device(device, res) else: res = [to_device(device, r) for r in res] result.append(res) return tuple(result)
def cgcnn_converter(batch, device=None, padding=None): """CGCNN converter""" if len(batch) == 0: raise ValueError("batch is empty") atom_feat, nbr_feat, nbr_idx = [], [], [] batch_atom_idx, target = [], [] current_idx = 0 xp = device.xp for element in batch: atom_feat.append(element[0]) nbr_feat.append(element[1]) nbr_idx.append(element[2] + current_idx) target.append(element[3]) n_atom = element[0].shape[0] atom_idx = numpy.arange(n_atom) + current_idx batch_atom_idx.append(atom_idx) current_idx += n_atom atom_feat = to_device(device, functions.concat(atom_feat, axis=0).data) nbr_feat = to_device(device, functions.concat(nbr_feat, axis=0).data) # Always use numpy array for batch_atom_index # this is list of variable length array batch_atom_idx = numpy.array(batch_atom_idx) nbr_idx = to_device(device, functions.concat(nbr_idx, axis=0).data) target = to_device(device, xp.asarray(target)) result = (atom_feat, nbr_feat, batch_atom_idx, nbr_idx, target) return result
def learn(self, batch_state_int, batch_action, batch_reward, batch_done, batch_next_state_int): batch_state_int, batch_action, batch_reward, batch_done, batch_next_state_int = to_device( self.network._device_id, (batch_state_int, batch_action, batch_reward, batch_done, batch_next_state_int)) batch_state = batch_state_int.astype(np.float32) / 255 # [0, 255] -> [0.0, 1.0] batch_next_state = batch_next_state_int.astype(np.float32) / 255 # [0, 255] -> [0.0, 1.0] batch_y, batch_q = self._compute_q_y(batch_state, batch_action, batch_reward, batch_done, batch_next_state) # assert len(batch_q.shape) == 1 # assert len(batch_y.shape) == 1 # assert batch_q.shape[0] == batch_y.shape[0] # loss = F.mean(F.huber_loss(batch_q, batch_y, delta=1.0, reduce='no')) loss = F.sum(F.huber_loss(batch_q, batch_y, delta=1.0, reduce='no')) with chainer.no_backprop_mode(): td_error = F.mean_absolute_error(batch_q, batch_y) self.network.cleargrads() loss.backward() self.optimizer.update() loss_cpu = to_device(CPU_ID, loss.array) td_error_cpu = to_device(CPU_ID, td_error.array) return loss_cpu, td_error_cpu
def __call__(self, state): one_batch_state = np.asarray([state], dtype=np.float32) one_batch_state = to_device(self.network._device_id, one_batch_state) with chainer.no_backprop_mode(): q_values = self.network(one_batch_state) q_values = to_device(CPU_ID, q_values.array)[0] action = np.argmax(q_values) return action, q_values
def converter(batch, device=-1): xs = [] ts = [] for b in batch: x = b[0] t = b[1] xs.append(to_device(device, x)) ts.append(to_device(device, t)) return (xs, ts)
def convert(batches, gpu_id=None): src = [] trg_sos = [] trg_eos = [] label = [] for batch in batches: src.append([to_device(gpu_id, b) for b in batch[0]]) trg_sos.append(to_device(gpu_id, batch[1])) trg_eos.append(to_device(gpu_id, batch[2])) label.append(to_device(gpu_id, batch[3])) return src, trg_sos, trg_eos, label
def convert(batch, gpu_id=None): articles = [] abstracts_sos = [] abstracts_eos = [] rule_flag_list = [] for data in batch: articles.append([to_device(gpu_id, sentence) for sentence in data[0]]) abstracts_sos.append([to_device(gpu_id, sentence) for sentence in data[1]]) abstracts_eos.append([to_device(gpu_id, sentence) for sentence in data[2]]) rule_flag_list.append(to_device(gpu_id, data[3])) return articles, abstracts_sos, abstracts_eos, rule_flag_list
def my_p_converter_pfeat(batch, device=None): Xp1 = [b[0] for b in batch] Xp2 = [b[1] for b in batch] Y = [b[2] for b in batch] Xp1 = np.vstack(Xp1) Xp2 = np.vstack(Xp2) Y = np.asarray(Y, np.int32)[:, None] Xp1 = to_device(device, Xp1) Xp2 = to_device(device, Xp2) Y = to_device(device, Y) return Xp1, Xp2, Y
def molder(self, ys): ys = [xp.reshape(xp.concatenate(y), (1, len(y))) for y in ys] ys = xp.concatenate(ys) ys = xp.hsplit(ys, ys.shape[1]) ys = [xp.concatenate(y) for y in ys] ys = to_device(self.device, ys) return ys
def my_converter(batch, device=None): Xim = [b[0] for b in batch] # # Comment out to enable data augmentation Xp1 = [b[1] for b in batch] Xp2 = [b[2] for b in batch] Y = [b[-1] for b in batch] Xp1 = np.vstack(Xp1) Xp2 = np.vstack(Xp2) Y = np.asarray(Y, np.int32)[:, None] Xp1 = to_device(device, Xp1) Xp2 = to_device(device, Xp2) Y = to_device(device, Y) return Xim, Xp1, Xp2, Y
def my_converter(batch, device=None): img = np.vstack([b[0] for b in batch]) img = to_device(device, img) actions = [b[1] for b in batch] actions = [item for sublist in actions for item in sublist] actions = [ to_device(device, np.asarray(x, dtype=np.int32)) for x in actions ] reasons = [b[2] for b in batch] reasons = [item for sublist in reasons for item in sublist] reasons = [ to_device(device, np.asarray(x, dtype=np.int32)) for x in reasons ] label = [b[3] for b in batch] label = to_device(device, np.asarray(label, dtype=np.int32).ravel()) ocr = [to_device(device, np.asarray(b[4]).astype('i')) for b in batch] return img, actions, reasons, label, ocr
def concat_examples(batch, device=None): # batch: img, mask, label, scale if len(batch) == 0: raise ValueError('batch is empty') first_elem = batch[0] result = [] for i in six.moves.range(len(first_elem)): array = _concat_arrays([example[i] for example in batch], None) if i == 0: # img result.append(to_device(device, array)) else: result.append(array) return tuple(result)
def convert_xt_batch_seq(xt_batch_seq, gpu): batchsize = len(xt_batch_seq[0]) seq_len = len(xt_batch_seq) xt_batch_seq = np.array(xt_batch_seq, 'i') # (bproplen, batch, 2) xt_batch_seq = convert.to_device(gpu, xt_batch_seq) xp = cuda.get_array_module(xt_batch_seq) x_seq_batch = xp.split( xt_batch_seq[:, :, 0].T.reshape(batchsize * seq_len), batchsize, axis=0) t_seq_batch = xp.split( xt_batch_seq[:, :, 1].T.reshape(batchsize * seq_len), batchsize, axis=0) return x_seq_batch, t_seq_batch
def concat_examples(batch, device=None, padding=None): if len(batch) == 0: raise ValueError('batch is empty') first_elem = batch[0] result = [] if not isinstance(padding, tuple): padding = [padding] * len(first_elem) for i in six.moves.range(len(first_elem)): res = _concat_arrays([example[i] for example in batch], padding[i]) if i in [0, 1]: # img, bbox res = to_device(device, res) result.append(res) return tuple(result)
def megnet_converter(batch, device=None, padding=0): """MEGNet converter""" if len(batch) == 0: raise ValueError("batch is empty") atom_feat, pair_feat, global_feat, target = [], [], [], [] atom_idx, pair_idx, start_idx, end_idx = [], [], [], [] batch_size = len(batch) current_atom_idx = 0 for i in range(batch_size): element = batch[i] n_atom = element[0].shape[0] n_pair = element[1].shape[0] atom_feat.extend(element[0]) pair_feat.extend(element[1]) global_feat.append(element[2]) atom_idx.extend([i]*n_atom) pair_idx.extend([i]*n_pair) start_idx.extend(element[3][0] + current_atom_idx) end_idx.extend(element[3][1] + current_atom_idx) target.append(element[4]) current_atom_idx += n_atom xp = device.xp atom_feat = to_device(device, xp.asarray(atom_feat)) pair_feat = to_device(device, xp.asarray(pair_feat)) global_feat = to_device(device, xp.asarray(global_feat)) atom_idx = to_device(device, xp.asarray(atom_idx)) pair_idx = to_device(device, xp.asarray(pair_idx)) start_idx = to_device(device, xp.asarray(start_idx)) end_idx = to_device(device, xp.asarray(end_idx)) target = to_device(device, xp.asarray(target)) result = (atom_feat, pair_feat, global_feat, atom_idx, pair_idx, start_idx, end_idx, target) return result
def converter(batch, gpu_id=None): text = [to_device(gpu_id, b) for b in batch[0]] label = to_device(gpu_id, batch[1]) return text, label
def convert_list(list_obj, gpu_id=None): return to_device(gpu_id, list_obj)