def encode_texts(self, texts, include_oov=False, verbose=1, **kwargs): """Encodes the given texts using internal vocabulary with optionally applied encoding options. See ``apply_encoding_options` to set various options. Args: texts: The list of text items to encode. include_oov: True to map unknown (out of vocab) tokens to 0. False to exclude the token. verbose: The verbosity level for progress. Can be 0, 1, 2. (Default value = 1) **kwargs: The kwargs for `token_generator`. Returns: The encoded texts. """ if not self.has_vocab: raise ValueError("You need to build the vocabulary using `build_vocab` before using `encode_texts`") progbar = Progbar(len(texts), verbose=verbose, interval=0.25) encoded_texts = [] for token_data in self.token_generator(texts, **kwargs): indices, token = token_data[:-1], token_data[-1] token_idx = self._token2idx.get(token) if token_idx is None and include_oov: token_idx = 0 if token_idx is not None: _append(encoded_texts, indices, token_idx) # Update progressbar per document level. progbar.update(indices[0]) # All done. Finalize progressbar. progbar.update(len(texts), force=True) return encoded_texts
def _load_text_word2vec(self, file_path): with open(file_path, 'r', encoding='utf-8') as f: line = f.readline() num_vectors, self.vector_len = (int(x) for x in line.split()) if self.verbose: self.logger.info( f'Loading {num_vectors} word vectors from file ' f'"{file_path}".') oov_id = self._init_vectors() found = 1 progbar = Progbar(len(self.vocab) - 1, verbose=self.verbose) for i in range(num_vectors): line = f.readline() values = line.split() word = values[0] word_vector = np.asarray(values[1:], dtype='float32') if word in self.vocab: word_id = self.vocab[word] self.vectors[word_id] = word_vector found += 1 progbar.update(found) if found < len(self.vocab): progbar.update(len(self.vocab) - 1) if oov_id is not None: self._replace_oov_tokens(oov_id)
def _test_loop(self, f, ins, batch_size=128, verbose=0): ''' Abstract method to loop over some data in batches. ''' nb_sample = len(ins[0]) outs = [] if verbose == 1: progbar = Progbar(target=nb_sample) batches = make_batches(nb_sample, batch_size) index_array = np.arange(nb_sample) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] ins_batch = slice_X(ins, batch_ids) batch_outs = f(*ins_batch) if type(batch_outs) == list: if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i, out in enumerate(outs): outs[i] /= nb_sample return outs
def epoch_end_callback(self, sess, sv, epoch_num): # Evaluate val loss validation_iou = 0 print("\nComputing Validation IoU") progbar = Progbar(target=self.val_steps_per_epoch) for i in range(self.val_steps_per_epoch): loss_iou = sess.run(self.val_iou, feed_dict={self.is_training: False}) validation_iou += loss_iou progbar.update(i) validation_iou /= self.val_steps_per_epoch * self.config.batch_size # Log to Tensorflow board val_sum = sess.run(self.val_sum, feed_dict={self.val_iou_ph: validation_iou}) sv.summary_writer.add_summary(val_sum, epoch_num) print("Epoch [{}] Validation IoU: {}".format(epoch_num, validation_iou)) # Model Saving if validation_iou > self.min_val_iou: self.save(sess, self.config.checkpoint_dir, 'best') self.min_val_iou = validation_iou if epoch_num % self.config.save_freq == 0: self.save(sess, self.config.checkpoint_dir, epoch_num)
def _predict_loop(self, f, ins, batch_size=128, verbose=0): ''' Abstract method to loop over some data in batches. ''' nb_sample = len(ins[0]) outs = [] if verbose == 1: progbar = Progbar(target=nb_sample) batches = make_batches(nb_sample, batch_size) index_array = np.arange(nb_sample) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] ins_batch = slice_X(ins, batch_ids) batch_outs = f(*ins_batch) if type(batch_outs) != list: batch_outs = [batch_outs] if batch_index == 0: for batch_out in batch_outs: shape = (nb_sample,) + batch_out.shape[1:] outs.append(np.zeros(shape)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) return outs
def test_progbar(): n = 2 input_arr = np.random.random((n, n, n)) bar = Progbar(n) for i, arr in enumerate(input_arr): bar.update(i, list(arr))
def load_subsets(self, subsets): """ Loads specified subsets of the data for the code jam. Returns tuple: ( images, labels, subset membership number ) You can use the subset membership number to select the data from particular subset: e.g. result[(indices == 4).flatten()] """ result = None resultLabels = None indices = None n_of_subsets = len(subsets) p = Progbar(n_of_subsets) p.update(0) for index, subsetIndex in enumerate(subsets): data = np.load("{}/{}.npz".format(self.root_path, subsetIndex)) if result is None: result = data['images'] else: result = np.vstack([result, data['images']]) if resultLabels is None: resultLabels = data['labels'] else: resultLabels = np.vstack([resultLabels, data['labels']]) tmp = np.ones(data['labels'].shape) * subsetIndex if indices is None: indices = tmp else: indices = np.vstack([indices, tmp]) p.update(index + 1) return (result, resultLabels, indices)
def test_extractor_in_generator(intervals, extractor, batch_size=128): """ Extracts data in bulk, then in streaming batches and checks its the same data. """ from keras.utils.generic_utils import Progbar X_in_memory = extractor(intervals) samples_per_epoch = len(intervals) batches_per_epoch = int(samples_per_epoch / batch_size) + 1 batch_array = np.zeros((batch_size, 1, 4, intervals[0].length), dtype=np.float32) batch_generator = generate_from_intervals(intervals, extractor, batch_size=batch_size, indefinitely=False, batch_array=batch_array) progbar = Progbar(target=samples_per_epoch) for batch_indx in xrange(1, batches_per_epoch + 1): X_batch = next(batch_generator) start = (batch_indx - 1) * batch_size stop = batch_indx * batch_size if stop > samples_per_epoch: stop = samples_per_epoch # assert streamed sequences and labels match data in memory assert (X_in_memory[start:stop] - X_batch).sum() == 0 progbar.update(stop)
def training(self, x_train_hw, y_train_hw, x_test, y_test): """Alternatively training models""" # get a batch of real images nb_train_hw = x_train_hw.shape[0] num_truncate = nb_train_hw % self.batch_size hw_data_used_num = nb_train_hw - num_truncate for epoch in range(self.nb_epochs): print('Epoch {} of {}'.format(epoch + 1, self.nb_epochs)) nb_batches = int(nb_train_hw / self.batch_size) progress_bar = Progbar(target=nb_batches) epoch_label_predictor_loss = [] for index in range(nb_batches): progress_bar.update(index) # get a batch of handwritten data hw_data_index_start = index * self.batch_size % hw_data_used_num hw_data_index_end = hw_data_index_start + self.batch_size img_hw = x_train_hw[hw_data_index_start:hw_data_index_end] cls_labels_hw = y_train_hw[ hw_data_index_start:hw_data_index_end] # updating parameters of label_predictor epoch_label_predictor_loss.append( self.character_classifier.train_on_batch([img_hw], [cls_labels_hw])) score = self.test(x_test, y_test) weights_output_dir = os.path.join( self.output_dir, 'pre_weights%02d-%04f.h5' % (epoch, score[1])) self.save_weights(weights_output_dir) print('\nTesting for epoch %02d: accuracy %04f' % (epoch + 1, score[1]))
def seq_to_preds(rows, seq_x, pred_probs, idx_to_attr_id, dilate=False): ''' rows: List of dicts, N elements seq_x: Tokens, N x L matrix pred_probs: Predicted idxs, N x L x C matrix ''' pred_labels = np.argmax(pred_probs, axis=2) # Set as -1 when x_i as PAD_IDX pad_idxs = np.where(seq_x == PAD_IDX) pred_labels[pad_idxs] = -1 predictions = [] n_rows = seq_x.shape[0] n_attr = len(idx_to_attr_id) progbar = Progbar(n_rows) for row_idx, row in enumerate(rows): row = rows[row_idx] gt_attr_ids = row['attr_id_seq'] rle_seq = row['rle_seq'] image_id = row['image_id'] pred_all_attr_idxs = pred_labels[ row_idx] # A sequence of size L, predictions of PAD_IDXs pred_seq_idxs = np.where(pred_all_attr_idxs >= 0)[0] pred_attr_idxs = pred_all_attr_idxs[pred_seq_idxs] pred_attr_ids = [idx_to_attr_id[a] for a in pred_attr_idxs] assert len(pred_attr_ids) == len(gt_attr_ids) for w_idx, pred_seq_idx in enumerate(pred_seq_idxs): for attr_idx in range(1, n_attr): # Exclude SAFE this_attr_id = idx_to_attr_id[attr_idx] this_score = pred_probs[row_idx, pred_seq_idx, attr_idx] w_rle = rle_seq[w_idx] if this_score > 1e-5 and w_rle is not None: if this_score > 0.05 and dilate: predictions.append({ 'image_id': image_id, 'attr_id': this_attr_id, 'segmentation': dilate_rle(w_rle), 'score': float(this_score), }) else: predictions.append({ 'image_id': image_id, 'attr_id': this_attr_id, 'segmentation': w_rle, 'score': float(this_score), }) progbar.update(row_idx) return predictions
def img_to_features(X, image_list, model, batch_size=64): n_img, n_h, n_w, n_c = X.shape n_batches = n_img / batch_size + 1 n_feat = model.output_shape[-1] feat_mat = np.zeros((n_img, n_feat)) pbar = Progbar(n_batches) for b_idx, start_idx in enumerate(range(0, n_img, batch_size)): end_idx = min(start_idx + batch_size, n_img) this_batch_size = end_idx - start_idx bx = X[start_idx:end_idx] bx = preprocess_input(bx) batch_feat = model.predict(bx) feat_mat[start_idx:end_idx] = batch_feat pbar.update(b_idx) # Create a dict: image_id -> feat image_id_to_visfeat = dict() for i, (image_id, image_path) in enumerate(image_list): image_id_to_visfeat[image_id] = feat_mat[i] return image_id_to_visfeat
def train_population(population, x, y, batch_size, steps, steps_save=100, validation_split=0.3): # Split data in train and validation. Set seed to get same splits in # consequent calls x_train, x_val, y_train, y_val = train_test_split( x, y, test_size=validation_split, random_state=42) population_size = len(population) batch_generator = BatchGenerator(x_train, y_train, batch_size) results = defaultdict(lambda: []) stateful_metrics = ['min_loss', 'max_loss', 'mean_loss'] for metric, _ in population[0].eval_metrics: stateful_metrics.extend( [m.format(metric) for m in ['min_{}', 'max_{}', 'mean_{}']]) progbar = Progbar(steps, stateful_metrics=stateful_metrics) for step in range(1, steps + 1): x, y = batch_generator.next() for idx, member in enumerate(population): # One step of optimisation using hyperparameters of 'member' member.step_on_batch(x, y) # Model evaluation loss = member.eval_on_batch(x_val, y_val) # If optimised for 'STEPS_READY' steps if member.ready(): # Use the rest of population to find better solutions exploited = member.exploit(population) # If new weights != old weights if exploited: # Produce new hyperparameters for 'member' member.explore() loss = member.eval_on_batch(x_val, y_val) if step % steps_save == 0 or step == steps: results['model_id'].append(str(member)) results['step'].append(step) results['loss'].append(loss) results['loss_smoothed'].append(member.loss_smoothed()) for metric, value in member.eval_metrics: results[metric].append(value) for h, v in member.get_hyperparameter_config().items(): results[h].append(v) # Get recently added losses to show in the progress bar all_losses = results['loss'] recent_losses = all_losses[-population_size:] if recent_losses: metrics = _statistics(recent_losses, 'loss') for metric, _ in population[0].eval_metrics: metrics.extend( _statistics(results[metric][-population_size:], metric)) progbar.update(step, metrics) return pd.DataFrame(results)
def _predict_loop(self, f, ins, batch_size=128, verbose=0): ''' Abstract method to loop over some data in batches. ''' nb_sample = len(ins[0]) outs = [] if verbose == 1: progbar = Progbar(target=nb_sample) batches = make_batches(nb_sample, batch_size) index_array = np.arange(nb_sample) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] ins_batch = slice_X(ins, batch_ids) batch_outs = f(*ins_batch) if type(batch_outs) != list: batch_outs = [batch_outs] if batch_index == 0: for batch_out in batch_outs: shape = (nb_sample, ) + batch_out.shape[1:] outs.append(np.zeros(shape)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) return outs
def image_list_to_arr(image_list): target_img_size = (250, 250) n_items = len(image_list) X = np.zeros(shape=(n_items, target_img_size[0], target_img_size[1], 3)) pbar = Progbar(n_items) for idx, (image_id, this_image_path) in enumerate(image_list): # ----- Image -> Mat resized_img_path = this_image_path.replace('images', 'images_250') resized_img_path = osp.join('/BS/orekondy2/work/datasets/VISPR2017', resized_img_path) if osp.exists(resized_img_path): this_image_path = resized_img_path else: this_image_path = osp.join(SEG_ROOT, this_image_path) img = load_img(this_image_path, target_size=target_img_size) img_arr = img_to_array(img) X[idx] = img_arr pbar.update(idx) return X
class TrainIntervalLogger(Callback): def __init__(self, interval=10000): self.interval = interval self.step = 0 self.reset() def reset(self): self.interval_start = timeit.default_timer() self.progbar = Progbar(target=self.interval) self.metrics = [] self.infos = [] self.info_names = None self.episode_rewards = [] def on_train_begin(self, logs): self.train_start = timeit.default_timer() self.metrics_names = self.model.metrics_names print('Training for {} steps ...'.format(self.params['nb_steps'])) def on_train_end(self, logs): duration = timeit.default_timer() - self.train_start print('done, took {:.3f} seconds'.format(duration)) def on_step_begin(self, step, logs): if self.step % self.interval == 0: if len(self.episode_rewards) > 0: metrics = np.array(self.metrics) assert metrics.shape == (self.interval, len(self.metrics_names)) formatted_metrics = '' if not np.isnan(metrics).all(): # not all values are means means = np.nanmean(self.metrics, axis=0) assert means.shape == (len(self.metrics_names),) for name, mean in zip(self.metrics_names, means): formatted_metrics += ' - {}: {:.3f}'.format(name, mean) formatted_infos = '' if len(self.infos) > 0: infos = np.array(self.infos) if not np.isnan(infos).all(): # not all values are means means = np.nanmean(self.infos, axis=0) assert means.shape == (len(self.info_names),) for name, mean in zip(self.info_names, means): formatted_infos += ' - {}: {:.3f}'.format(name, mean) print('{} episodes - episode_reward: {:.3f} [{:.3f}, {:.3f}]{}{}'.format(len(self.episode_rewards), np.mean(self.episode_rewards), np.min(self.episode_rewards), np.max(self.episode_rewards), formatted_metrics, formatted_infos)) print('') self.reset() print('Interval {} ({} steps performed)'.format(self.step // self.interval + 1, self.step)) def on_step_end(self, step, logs): if self.info_names is None: self.info_names = logs['info'].keys() values = [('reward', logs['reward'])] self.progbar.update((self.step % self.interval) + 1, values=values, force=True) self.step += 1 self.metrics.append(logs['metrics']) if len(self.info_names) > 0: self.infos.append([logs['info'][k] for k in self.info_names]) def on_episode_end(self, episode, logs): self.episode_rewards.append(logs['episode_reward'])
def _save_predictions_to_xmls(model, batch_size, embeddings, label2ind, ind2label, test_set, predictions_dir, binary_classification, hipaa_only, extra_features, require_argmax): if not os.path.isdir(predictions_dir): os.mkdir(predictions_dir) print('Saving test XMLs to', predictions_dir) progress_bar = Progbar(target=TestSet.number_of_test_sets(test_set), verbose=env.keras_verbose) for i, te in enumerate(TestSet.test_sets( embeddings, test_set=test_set, label2ind=label2ind, binary_classification=binary_classification, hipaa_only=hipaa_only, extra_features=extra_features), start=1): preds = model.predict([te.X, te.X_extra], batch_size=batch_size) if require_argmax: preds = np.argmax(preds, axis=-1) xml = prediction_to_xml(te.X, preds, te.text, te.sents, ind2label) filename = os.path.basename(te.filename)[:-4] + '.xml' with open(os.path.join(predictions_dir, filename), 'w') as f: f.write(xml) progress_bar.update(i)
def run_train_epoch(self, session, x_inputs, batch_size, shuffle=True, verbose=1): num_samples = len(x_inputs) index_array = np.arange(num_samples) if shuffle: np.random.shuffle(index_array) batches = self.make_batches(num_samples, batch_size) nb_batch = len(batches) progbar = Progbar(nb_batch) avg_total_loss = 0. total_samples = 0. for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] x_batch = x_inputs[batch_ids] _, loss = session.run([self.train_op, self.loss], {self.x: x_batch}) if np.isnan(loss) or np.isinf(loss): raise ValueError("nan or inf loss") cur_batch_size = (batch_end - batch_start) total_samples += cur_batch_size avg_total_loss += (loss * cur_batch_size / num_samples) if verbose == 1: progbar.update(batch_index + 1, values=[("avg loss per 1000 samples", avg_total_loss * 1000. / total_samples) ], force=True) print("avg total loss = %d" % avg_total_loss)
def evaluate_by_datasets(self, model): from keras.utils.generic_utils import Progbar results = [] for i, single in enumerate(self.single_datasets): ys = [np.zeros(s.y_valid.shape[1:]) for s in self.single_datasets] # makes blank ys result = [] print('Evaluating', single.name) progbar = Progbar(len(single.X_valid)) for j in range(len(single.X_valid)): X, y = next(single.valid_generator) Xtemp = [] for x_one in X: x_normed = util.random_unify_3d_mels(x_one, self.duration) Xtemp.append(x_normed) Xtemp = np.array(Xtemp) result.append( np.argmax(y) == np.argmax(model.predict(Xtemp)[i])) progbar.update(j) results.append(result) progbar.update(len(single.X_valid)) print(' =', np.sum(result) / len(result)) accuracies = [np.sum(result) / len(result) for result in results] for s, acc in zip(self.single_datasets, accuracies): print('Accuracy with %s = %f' % (s.name, acc)) return accuracies
def build_vocab(self, texts, verbose=1, **kwargs): """Builds the internal vocabulary and computes various statistics. Args: texts: The list of text items to encode. verbose: The verbosity level for progress. Can be 0, 1, 2. (Default value = 1) **kwargs: The kwargs for `token_generator`. """ if self.has_vocab: logger.warn( "Tokenizer already has existing vocabulary. Overriding and building new vocabulary." ) progbar = Progbar(len(texts), verbose=verbose, interval=0.25) count_tracker = _CountTracker() self._token_counts.clear() self._num_texts = len(texts) for token_data in self.token_generator(texts, **kwargs): indices, token = token_data[:-1], token_data[-1] count_tracker.update(indices) self._token_counts[token] += 1 # Update progressbar per document level. progbar.update(indices[0]) # Generate token2idx and idx2token. self.create_token_indices(self._token_counts.keys()) # All done. Finalize progressbar update and count tracker. count_tracker.finalize() self._counts = count_tracker.counts progbar.update(len(texts), force=True)
def trainer_on_batch(model, train_x, train_y, batch_size=_default_batch_size, epochs=_default_epochs): for epoch in range(epochs): print('Epoch {} of {}'.format(epoch + 1, epochs)) nb_batches = int(train_x.shape[0] / batch_size) # progress_bar display progress_bar = Progbar(target=train_x.shape[0]) batch_res = [None, None] history = [] start_epoch = time.time() for iter in range(nb_batches): # get a batch train_set train_x_batch = train_x[iter * batch_size:(iter + 1) * batch_size] train_y_batch = train_y[iter * batch_size:(iter + 1) * batch_size] batch_res = model.train_on_batch(x=train_x_batch, y=train_y_batch) history.append(batch_res) # update the progress_bar progress_bar.update((iter + 1) * batch_size) end_epoch = time.time() print(' epoch_loss: {} epoch_acc: {} epoch_time:{}'.format( str(batch_res[0]), str(batch_res[1]), end_epoch - start_epoch)) return model, history
def fit_model(self, X, y): """ fits a model to some data """ for e in range(self.nb_epoch): print('Epoch: ', e, ' of ', self.nb_epoch) progbar = Progbar(target=X.shape[0], verbose=True) # batch train with realtime data augmentation total_accuracy = 0 total_loss = 0 current = 0 for X_batch, y_batch in self.datagen.flow(X, y, self.batch_size): # prepare the batch with random augmentations X_batch, y_batch = self.batch_warp(X_batch, y_batch) # train on the batch loss, accuracy = self.model.train(X_batch, y_batch, accuracy = True) # update the progress bar total_loss += loss * self.batch_size total_accuracy += accuracy * self.batch_size current += self.batch_size if current > self.X.shape[0]: current = self.X.shape[0] else: progbar.update(current, [('loss', loss), ('acc.', accuracy)]) progbar.update(current, [('loss', total_loss/current), ('acc.', total_accuracy/current)]) # checkpoints between epochs self.model.save_weights(self.save_weights_file, overwrite = True)
def test(self, model, queue, batch_size=1000, verbose=True, test_size=None): iterator = None process = psutil.Process(os.getpid()) def get_rss_prop(): # this is quite expensive return (process.memory_info().rss - process.memory_info().shared) / 10**6 rss_minus_shr_memory = get_rss_prop() try: iterator = gf_io_utils.ExampleQueueIterator( queue, num_exs_batch=batch_size, num_epochs=1, allow_smaller_final_batch=True) if test_size is not None: num_examples = min(test_size, iterator.num_examples) else: num_examples = iterator.num_examples num_batches = int(np.ceil(num_examples / batch_size)) if verbose: progbar = Progbar(target=num_examples) predictions = [] labels = [] for batch_indx, batch in enumerate(iterator): if batch_indx == num_batches: break predictions.append( np.vstack(model.model.predict_on_batch(batch))) labels.append(batch['labels']) if verbose: if batch_indx % BATCH_FREQ_UPDATE_MEM_USAGE == 0: rss_minus_shr_memory = get_rss_prop() if batch_indx % BATCH_FREQ_UPDATE_PROGBAR == 0: progbar.update(batch_indx * batch_size, values=[("Non-shared RSS (Mb)", rss_minus_shr_memory)]) iterator.close() del iterator except Exception as e: if iterator is not None: # NOQA iterator.close() # NOQA raise e predictions = np.vstack(predictions) labels = np.vstack(labels) return ClassificationResult(labels, predictions, task_names=self.task_names)
def train_model(train_X, val_X, mu, sigma): assert train_X.ndim == 3, train_X.shape total_X, time_steps, out_shape = train_X.shape trainer = GANTrainer(out_shape) epochs = 0 # GAN predictions will be put in here try: mkdir('gan-conv-out') except FileExistsError: pass print('Training generator') while True: copy_X = train_X.copy() np.random.shuffle(copy_X) total_X, _, _ = copy_X.shape to_fetch = BATCH_SIZE // 2 epochs += 1 print('Epoch %d' % epochs) bar = Progbar(total_X) bar.update(0) epoch_fetched = 0 while epoch_fetched < total_X: # Fetch some ground truth to train the discriminator for i in range(K): if epoch_fetched >= total_X: break fetched = copy_X[epoch_fetched:epoch_fetched + to_fetch] dloss, dacc = trainer.disc_train_step(fetched) epoch_fetched += len(fetched) bar.update(epoch_fetched, values=[('d_loss', dloss), ('d_acc', dacc)]) # Train the generator (don't worry about loss) trainer.gen_train_step(BATCH_SIZE) # End of an epoch, so let's validate models (doesn't work so great, # TBH) print('\nValidating') disc_loss, disc_acc = trainer.disc_val(val_X, BATCH_SIZE) gen_loss, gen_acc = trainer.gen_val(100, BATCH_SIZE) print('\nDisc loss/acc: %g/%g' % (disc_loss, disc_acc)) print('Gen loss/acc: %g/%g' % (gen_loss, gen_acc)) # Also save some predictions so that we can monitor training print('Saving predictions') poses = trainer.generate_poses(16) * sigma + mean poses = insert_junk_entries(poses) savemat('gan-conv-out/gan-conv-preds-epoch-%d.mat' % epochs, {'poses': poses}) # Sometimes we save a model if not (epochs - 1) % 5: dest_dir = 'saved-conv-gans/' print('Saving model to %s' % dest_dir) trainer.save(dest_dir)
def train_model(self): cbs = [] cbs.append(EarlyStopping(patience=2)) cbs.append(LearningRateScheduler(lambda e: self.lr * 0.999**(e / 10))) cb = CallBacks(cbs) cb.set_model(self.model) print('Start training chatbot...') train_num = len(self.en_ipt) cb.on_train_begin() for itr in range(self.epoch): print('Epoch %s/%s' % (itr + 1, self.epoch)) cb.on_epoch_begin(itr) indexes = np.random.permutation(train_num) progbar = Progbar(train_num) losses = [] for idx in range(int(0.8 * train_num / self.bs)): batch_idx = indexes[idx * self.bs:(idx + 1) * self.bs] en_ipt_bc = self.en_ipt[batch_idx] de_ipt_bc = self.de_ipt[batch_idx] de_opt_bc = self.de_opt[batch_idx] if np.random.rand() < self.tfr: # apply teacher forcing bc_loss = self.model.train_on_batch([en_ipt_bc, de_ipt_bc], de_opt_bc) else: # do not apply teacher forcing ipt_len = [sum(i) for i in np.any(de_opt_bc, axis=-1)] de_ipt_nt = np.zeros((self.max_de_seq, self.bs), dtype='int64') en_out, h, c = self.encoder_model.predict( en_ipt_bc, batch_size=self.bs) de_in = np.asarray([[self.word2idx['bos']]] * self.bs) for i in range(self.max_de_seq): de_out, h, c = self.decoder_model.predict( [en_out, de_in, h, c], batch_size=self.bs) sampled_idxs = np.argmax(de_out[:, -1, :], axis=-1) de_ipt_nt[i] = sampled_idxs de_in = sampled_idxs.reshape((-1, 1)) de_ipt_nt = de_ipt_nt.T for i in range(self.bs): de_ipt_nt[i, ipt_len[i]:] = 0 bc_loss = self.model.train_on_batch([en_ipt_bc, de_ipt_nt], de_opt_bc) losses.append(bc_loss) progbar.add(self.bs, [('loss', np.mean(losses))]) val_idx = indexes[-int(0.2 * train_num):] val_loss = self.model.evaluate( [self.en_ipt[val_idx], self.de_ipt[val_idx]], self.de_opt[val_idx], batch_size=self.bs, verbose=0) progbar.update(train_num, [('val_loss', np.mean(val_loss))]) cb.on_epoch_end(itr, logs={ 'loss': np.mean(losses), 'val_loss': np.mean(val_loss) }) self.model.save_weights(self.ckpt_dir + 'weights.hdf5') cb.on_train_end() print('Chatbot training complete.')
def predict(model, generator, steps): prog = Progbar(steps) preds = [] for i, batch in enumerate(generator): preds.append(model.predict_on_batch(batch)) prog.update(i + 1) print("") return preds
def train(self, epochs, batch_size, X_train, save_result_dir, save_weight_dir): g_losses = [] d_losses = [] # noiseの発生を一定にする。 # create random noise -> N latent vectors np.random.seed(0) noise = np.random.uniform(-1, 1, size=(batch_size, self.input_dim)) # np.save('seed50.npy', noise) # epoch開始 for epoch in range(epochs): np.random.shuffle(X_train) # traindataをbatchsize分に分割するので、iter = train//batch n_iter = X_train.shape[0] // batch_size progress_bar = Progbar(target=n_iter) # イテレーション開始 for index in range(n_iter): # load real data & generate fake data image_batch = X_train[index * batch_size:(index + 1) * batch_size] for i in range(batch_size): if np.random.random() > 0.5: image_batch[i] = np.fliplr(image_batch[i]) if np.random.random() > 0.5: image_batch[i] = np.flipud(image_batch[i]) generated_images = self.g.predict(noise, verbose=0) # print(generated_images.shape) # attach label for training discriminator X = np.concatenate((image_batch, generated_images)) y = np.array([1] * batch_size + [0] * batch_size) # training discriminator d_loss = self.d.train_on_batch(X, y) # training generator g_loss = self.dcgan.train_on_batch(noise, np.array([1] * batch_size)) progress_bar.update(index, values=[('g', g_loss), ('d', d_loss)]) g_losses.append(g_loss) d_losses.append(d_loss) image = self.combine_images(generated_images) image = (image + 1) / 2.0 * 255.0 cv2.imwrite(os.path.join(save_result_dir, str(epoch) + ".png"), image) print('\nEpoch' + str(epoch) + " end") # save weights for each epoch if (epoch + 1) % 50 == 0: self.g.save_weights( os.path.join(save_weight_dir, "generator_" + str(epoch) + '.h5'), True) self.d.save_weights( os.path.join(save_weight_dir, "discriminator_" + str(epoch) + '.h5'), True) return g_losses, d_losses
def train(BATCH_SIZE, X_train, n_EPOCH): ### model define d = discriminator_model() g = generator_model() #d.summary() #g.summary() d_on_g = generator_containing_discriminator(g, d) d_optim = RMSprop(lr=0.0004) g_optim = RMSprop(lr=0.0002) g.compile(loss='mse', optimizer=g_optim) d_on_g.compile(loss='mse', optimizer=g_optim) d.trainable = True d.compile(loss='mse', optimizer=d_optim) for epoch in range(n_EPOCH): print("Epoch is", epoch) n_iter = int(X_train.shape[0] / BATCH_SIZE) progress_bar = Progbar(target=n_iter) for index in range(n_iter): # create random noise -> U(0,1) 10 latent vectors noise = np.random.uniform(0, 1, size=(BATCH_SIZE, 10)) # load real data & generate fake data image_batch = X_train[index * BATCH_SIZE:(index + 1) * BATCH_SIZE] generated_images = g.predict(noise, verbose=0) # visualize training results if index % 20 == 0: image = combine_images(generated_images) image = image * 127.5 + 127.5 cv2.imwrite( './result/' + str(epoch) + "_" + str(index) + ".png", image) # attach label for training discriminator #print("image_batch shape", image_batch.shape) #print("generated_images shape", generated_images.shape) X = np.concatenate((image_batch, generated_images)) y = np.array([1] * BATCH_SIZE + [0] * BATCH_SIZE) # training discriminator d_loss = d.train_on_batch(X, y) # training generator d.trainable = False g_loss = d_on_g.train_on_batch(noise, np.array([1] * BATCH_SIZE)) d.trainable = True progress_bar.update(index, values=[('g', g_loss), ('d', d_loss)]) print('') # save weights for each epoch g.save_weights('weights/generator.h5', True) d.save_weights('weights/discriminator.h5', True) return d, g
def input_fn(tf_glob, one_hot=True, classes=None, is_training=None, batch_shape=[32, 224, 224, 3], parallelism=1): """ Return tensor to read from TFRecord """ print('Creating graph for loading %s TFRecords...' % tf_glob) with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0], parallelism=parallelism) records_op = record_input.get_yield_op() records_op = tf.split(records_op, batch_shape[0], 0) records_op = [tf.reshape(record, []) for record in records_op] progbar = Progbar(len(records_op)) images = [] labels = [] for i, serialized_example in enumerate(records_op): progbar.update(i) with tf.variable_scope("parse_images", reuse=True): features = tf.parse_single_example( serialized_example, features={ 'image': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), }) image_decoded = tf.image.decode_jpeg(features['image'], channels=3) image = tf.image.convert_image_dtype(image_decoded, tf.float32) resized_image = tf.image.resize_images( image, [batch_shape[1], batch_shape[2]]) label = tf.cast(features['label'], tf.int32) if one_hot and classes: label = tf.one_hot(label, classes) images.append(resized_image) labels.append(label) images = tf.parallel_stack(images, 0) labels = tf.parallel_stack(labels, 0) # images = tf.cast(images, tf.float32) # images = tf.reshape(images, shape=batch_shape) # StagingArea will store tensors # across multiple steps to # speed up execution images_shape = images.get_shape() labels_shape = labels.get_shape() copy_stage = data_flow_ops.StagingArea( [tf.float32, tf.float32], shapes=[images_shape, labels_shape]) copy_stage_op = copy_stage.put([images, labels]) staged_images, staged_labels = copy_stage.get() return images, labels
def train(run_name, digit, nepochs, batch_size, latent_dim, noise_type, k, optimizer, model_dir, gen_intermediate_dims=None, disc_intermediate_dims=None): # load data and params Xtr, Xte, _, _ = load_data(digit) Xte = Xte[:(Xte.shape[0] / batch_size) * batch_size] # correct for batch_size original_dim = Xtr.shape[-1] nbatches = int(Xtr.shape[0] / batch_size) fnm_gen, fnm_disc, fnm_hist, fnm_samp = get_filenames(model_dir, run_name) # load models generator = build_generator(batch_size, latent_dim, gen_intermediate_dims, original_dim, optimizer) discriminator = build_discriminator(2 * batch_size, original_dim, disc_intermediate_dims, optimizer) combined = build_combined(2 * batch_size, latent_dim, generator, discriminator, optimizer) # train and test history = init_history() for i in xrange(nepochs): print('Epoch {} of {}'.format(i + 1, nepochs)) progress_bar = Progbar(target=nbatches) # train on mini-batches of train data epoch_losses = [] for j in xrange(nbatches): progress_bar.update(j) batch_loss = train_on_batch(Xtr, j, batch_size, latent_dim, noise_type, k, generator, discriminator, combined) epoch_losses.append(batch_loss) # evaluate on test data print('\nTesting epoch {}:'.format(i + 1)) test_loss = evaluate(Xte, batch_size, latent_dim, noise_type, generator, discriminator, combined, fnm_samp.format(i + 1)) train_loss = np.mean(np.array(epoch_losses), axis=0) history = update_history(history, train_loss, test_loss, do_print=True) # save weights generator.save_weights(fnm_gen.format(i), True) discriminator.save_weights(fnm_disc.format(i), True) # save history save_history(fnm_hist, history)
def _test_masks(): assert os.path.exists(FLAGS.filename) assert os.path.exists(FLAGS.flow_dir) assert os.path.exists(FLAGS.root_dir) learner = AdversarialLearner() learner.setup_inference(FLAGS, aug_test=False) saver = tf.train.Saver([var for var in tf.trainable_variables()]) # manages multi-threading sv = tf.train.Supervisor(logdir=FLAGS.test_save_dir, save_summaries_secs=0, saver=None) with open(FLAGS.filename, 'r') as f: samples = f.readlines() with sv.managed_session() as sess: checkpoint = FLAGS.ckpt_file if checkpoint: saver.restore(sess, checkpoint) print("Resume model from checkpoint {}".format(checkpoint)) else: raise IOError("Checkpoint file not found") sess.run(learner.test_iterator.initializer) n_steps = len(samples) progbar = Progbar(target=n_steps) i = 0 for step in range(n_steps): if sv.should_stop(): break try: inference = learner.inference(sess) except tf.errors.OutOfRangeError: print("End of testing dataset") # ==> "End of dataset" break # Now write images in the test folder # select mask generated_mask = inference['mask'] # Verbose image generation save_dir = os.path.join( FLAGS.test_save_dir, os.path.dirname(samples[step].split(' ')[0])) if not os.path.exists(save_dir): os.makedirs(save_dir) filename = os.path.join( save_dir, os.path.basename(samples[step].split(' ')[0])) cv2.imwrite(filename, generated_mask.squeeze() * 255.) i += 1 progbar.update(step)
def train(generator, discriminator, combined, epochs): # Load and normalize data: (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = np.concatenate((x_train, x_test), axis=0) x_train = x_train.reshape((NUM_IMGS, IMG_ROWS, IMG_COLS, CHANNELS)) y_train = np.concatenate((y_train, y_test), axis=0) y_train = y_train.reshape((-1, 1)) x_train = (x_train.astype(np.float32) - 127.5) / 127.5 # Label arrays for positive/negative examples positive_examples = np.ones((BATCH_SIZE, 1)) negative_examples = np.zeros((BATCH_SIZE, 1)) # Number of batch loops: batch_loops = int(NUM_IMGS // BATCH_SIZE) # Train cGAN: for epoch in range(epochs): progress_bar = Progbar(target=batch_loops) shuffle_idx = np.random.permutation(NUM_IMGS) real_imgs = x_train[shuffle_idx] labels = y_train[shuffle_idx] for batch_i in range(batch_loops): progress_bar.update(batch_i) # Discriminator: img_batch = real_imgs[batch_i * BATCH_SIZE:(batch_i + 1) * BATCH_SIZE] label_batch = labels[batch_i * BATCH_SIZE:(batch_i + 1) * BATCH_SIZE] noise = np.random.normal(0, 1, (BATCH_SIZE, LATENT_SIZE)) fake_img_batch = generator.predict([noise, label_batch]) d_loss_real = discriminator.train_on_batch( [img_batch, label_batch], positive_examples) d_loss_fake = discriminator.train_on_batch( [fake_img_batch, label_batch], negative_examples) d_loss_total = 0.5 * np.add(d_loss_real, d_loss_fake) # Generator: noise = np.random.normal(0, 1, (2 * BATCH_SIZE, LATENT_SIZE)) fake_labels = np.random.randint(0, NUM_CLASSES, (2 * BATCH_SIZE, 1)) positive = np.concatenate((positive_examples, positive_examples), axis=0) g_loss = combined.train_on_batch([noise, fake_labels], positive) print("Epoch: %d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss_total[0], 100 * d_loss_total[1], g_loss)) save_images(generator, epoch) return generator
def predict(self, model, queue, batch_size=1000, verbose=True): iterator = None process = psutil.Process(os.getpid()) def get_rss_prop(): # this is quite expensive return (process.memory_info().rss - process.memory_info().shared) / 10**6 rss_minus_shr_memory = get_rss_prop() try: iterator = gf_io_utils.ExampleQueueIterator( queue, num_exs_batch=batch_size, num_epochs=1, allow_smaller_final_batch=True) if verbose: progbar = Progbar(target=iterator.num_examples) chroms = [] starts = [] ends = [] predictions = [] for batch_indx, batch in enumerate(iterator): chroms.append(batch['intervals/chrom']) starts.append(batch['intervals/start']) ends.append(batch['intervals/end']) predictions.append( np.vstack(model.model.predict_on_batch(batch))) if verbose: if batch_indx % BATCH_FREQ_UPDATE_MEM_USAGE == 0: rss_minus_shr_memory = get_rss_prop() if batch_indx % BATCH_FREQ_UPDATE_PROGBAR == 0: progbar.update(batch_indx * batch_size, values=[("Non-shared RSS (Mb)", rss_minus_shr_memory)]) iterator.close() del iterator except Exception as e: if iterator is not None: # NOQA iterator.close() # NOQA raise e # concatenate intervals and predictions intervals = { 'chrom': np.concatenate(chroms), 'start': np.concatenate(starts), 'end': np.concatenate(ends) } predictions = np.vstack(predictions) return intervals, predictions
def test_progbar(): values_s = [None, [['key1', 1], ['key2', 1e-4]], [['key3', 1], ['key2', 1e-4]]] for target in (len(values_s) - 1, None): for verbose in (0, 1, 2): bar = Progbar(target, width=30, verbose=verbose, interval=0.05) for current, values in enumerate(values_s): bar.update(current, values=values)
class TrainIntervalLogger(Callback): def __init__(self, interval=10000): self.interval = interval self.step = 0 self.reset() def reset(self): """ Reset statistics """ self.interval_start = timeit.default_timer() self.progbar = Progbar(target=self.interval) self.metrics = [] self.infos = [] self.info_names = None self.episode_rewards = [] def on_train_begin(self, logs): """ Initialize training statistics at beginning of training """ self.train_start = timeit.default_timer() self.metrics_names = metrics_names() print('Training for {} steps ...'.format(self.params['nb_steps'])) def on_train_end(self, logs): """ Print training duration at end of training """ duration = timeit.default_timer() - self.train_start print('done, took {:.3f} seconds'.format(duration)) def on_step_begin(self, step, logs): """ Print metrics if interval is over """ if self.step % self.interval == 0: if len(self.episode_rewards) > 0: metrics = np.array(self.metrics) assert metrics.shape == (self.interval, len(self.metrics_names)) formatted_metrics = '' if not np.isnan(metrics).all(): # not all values are means means = np.nanmean(self.metrics, axis=0) assert means.shape == (len(self.metrics_names),) for name, mean in zip(self.metrics_names, means): formatted_metrics += ' - {}: {:.3f}'.format(name, mean) formatted_infos = '' print('{} episodes - episode_reward: {:.3f} [{:.3f}, {:.3f}]{}{}'.format(len(self.episode_rewards), np.mean(self.episode_rewards), np.min(self.episode_rewards), np.max(self.episode_rewards), formatted_metrics, formatted_infos)) print('') self.reset() print('Interval {} ({} steps performed)'.format(self.step // self.interval + 1, self.step)) def on_step_end(self, step, logs): """ Update progression bar at the end of each step """ values = [('reward', logs['reward'])] self.progbar.update((self.step % self.interval) + 1, values=values) self.step += 1 self.metrics.append(logs['metrics']) def on_episode_end(self, episode, logs): """ Update reward value at the end of each episode """ self.episode_rewards.append(logs['episode_reward'])
def predict_general_(self, model, X, size, load_func): queue = Queue.Queue() #generate the progress bar if self.verbose>0: progbar = Progbar(size, width=80, verbose=self.verbose) batch_idx = range(min(size, self.memory_batch_size)); self.matrix_load_into_queue(X, batch_idx, queue, load_func); X_batch,_,_ = queue.get() p = [] samples = 0 last_update = time.time()-1000 for _, i in enumerate(xrange(0, size, self.memory_batch_size)): next_start = i+len(batch_idx) next_end = min(size, next_start+self.memory_batch_size) if next_end>next_start: #spin the thread up batch_idx_next = range(next_start,next_end); thread = threading.Thread(target=self.matrix_load_into_queue, args=(X,batch_idx_next,queue,load_func)) thread.start() else: batch_idx_next = None thread = None #predict the value if X_batch.shape[0]>0: p_curr = model.predict(X_batch, batch_size=self.batch_size, verbose=0) p.append(p_curr) #increment the counter samples+= len(batch_idx) curr_update = time.time() if self.verbose>0 and (curr_update-last_update>=0.5 or (samples)>=size): progbar.update(samples, []) last_update = curr_update #wait for the next load to happen if thread is not None: thread.join() X_batch,_,_ = queue.get() #now add the next batch batch_idx = batch_idx_next p = np.vstack(p) return p
def positive(self, set_positive=1.): """ Create a list of positive data points, expand them to square disk of radius `self.radius` pixels. :param set_positive: Set positive to max(1., p) where p is given probability :return: Dictionary(key: filename, value: [x, y, (p, 1-p)]) """ if self.positives is not None: # If already calculated then return it. return self.positives, self.positives_count if self.files is None: # Curate list of files if not done already. self.files = read_all_files(self.path) bar = Progbar(len(self.files)) # Create instance of progress bar. if self.verbose: # Verbose output for debugging. print TT.info("> Collecting positive samples from dataset...") bar.update(0) index = 0 # File index - to update state of progress bar. count = 0 # Holds total number of positive samples. expanded = {} # Holds list of files and positive pixels in flattened image with mitosis probability. normal = {} # Holds list of files and positive pixel (y, x) along with class probabilities. # (0: Mitotic, 1: Non-Mitotic) total = 0 for data_image, target_csv in self.files: labels = csv2np(os.path.join(self.path, target_csv)) # Load CSV annotations into numpy array. expanded[data_image] = {} # Initialize list for file normal[data_image] = [] total += len(labels) for (y, x, p) in labels: # Iterate over annotated pixel values. x = int(x) y = int(y) p = max(set_positive, float(p)) # Image position, horizontal -> y, vertical -> x # Image size, (y, x) # @see http://www.scipy-lectures.org/advanced/image_processing/#basic-manipulations range_x = xrange(max(0, x - self.radius), min(x + self.radius, self.image_size[1])) range_y = xrange(max(0, y - self.radius), min(y + self.radius, self.image_size[0])) for i in range_x: for j in range_y: expanded[data_image][i * self.image_size[0] + j] = p # TODO: Verify this. `x * width + y` normal[data_image].append([i, j, p]) # (x, y) => (row, column) count += 1 index += 1 if self.verbose: bar.update(index) self.positives = normal self.positives_sorted = expanded self.positives_count = count TT.success("> Total", count, "positive pixels from", total, "annotations.") return normal, count
class TrainIntervalLogger(Callback): def __init__(self, interval=10000): self.interval = interval self.step = 0 self.reset() def reset(self): self.interval_start = timeit.default_timer() self.progbar = Progbar(target=self.interval) self.metrics = [] def on_train_begin(self, logs): self.train_start = timeit.default_timer() self.metrics_names = self.model.metrics_names print('Training for {} steps ...'.format(self.params['nb_steps'])) def on_train_end(self, logs): duration = timeit.default_timer() - self.train_start print('done, took {:.3f} seconds'.format(duration)) def on_step_begin(self, step, logs): if self.step % self.interval == 0: self.reset() print('Interval {} ({} steps performed)'.format(self.step / self.interval + 1, self.step)) def on_step_end(self, step, logs): # TODO: work around nan's in metrics. This isn't really great yet and probably not 100% accurate filtered_metrics = [] means = None for idx, value in enumerate(logs['metrics']): if not np.isnan(value): filtered_metrics.append(value) else: mean = np.nan if len(self.metrics) > 0 and not np.isnan(self.metrics).all(): if means is None: means = np.nanmean(self.metrics, axis=0) assert means.shape == (len(self.metrics_names),) mean = means[idx] filtered_metrics.append(mean) values = [('reward', logs['reward'])] if not np.isnan(filtered_metrics).any(): values += list(zip(self.metrics_names, filtered_metrics)) self.progbar.update((self.step % self.interval) + 1, values=values, force=True) self.step += 1 self.metrics.append(logs['metrics'])
def run(tag_dist, output_fname, force, nb_samples): os.makedirs(os.path.dirname(output_fname), exist_ok=True) if os.path.exists(output_fname) and force: print("Deleted {}".format(output_fname)) os.remove(output_fname) else: assert not os.path.exists(output_fname), \ "File {} already exists. Use --force to override it" basename, _ = os.path.splitext(output_fname) anit_name = basename + "_anti_{}.png" hist_name = basename + "_hist_{}.png" plot_anitaliasing(tag_dist, anit_name, 1) plot_anitaliasing(tag_dist, anit_name, 2) plot_anitaliasing(tag_dist, anit_name, 4) plot_anitaliasing(tag_dist, anit_name, 8) labels, masks, _ = next(generator(tag_dist, 10000, antialiasing=2)) for key in labels.dtype.names: m = labels[key].mean() s = labels[key].std() print("{}: {:.3f}, {:.3f}".format(key, m, s)) assert abs(m) <= 0.03 for label_name in sorted(set(labels.dtype.names) - set(['bits'])): x = labels[label_name] plt.hist(x.flatten(), bins=40, normed=True) plt.savefig(hist_name.format(label_name)) plt.clf() dset = DistributionHDF5Dataset(output_fname, distribution=tag_dist, nb_samples=nb_samples, mode='w') progbar = Progbar(nb_samples) batch_size = min(25000, nb_samples) for labels, tags, depth_map in generator(tag_dist, batch_size, antialiasing=4): pos = dset.append(labels=labels, tag3d=tags, depth_map=depth_map) progbar.update(pos) if pos == nb_samples: break print("Saved tag 3d dataset to: {}".format(output_fname)) dist_fname = basename + "_distribution.json" with open(dist_fname, "w+") as dist_f: dist_f.write(tag_dist.to_json()) print("Saved distribution to: {}".format(dist_fname))
def sample(self, batch_size=100): if self.sampled is not None and self.batch == batch_size: return self.sampled, batch_size self.batch = batch_size self.positive() if self.verbose: TT.info("> Creating a random dataset...") if self.files is None: self.files = read_all_files(self.path) TT.info("> Sampling from", len(self), "pixels.") indices = xrange(len(self)) sampled = {} bar = Progbar(self.batch) count = 0 positives = 0 if self.verbose: bar.update(count) for index in random.sample(indices, self.batch): file_id = index / self.pixels_per_image image, csv = self.files[file_id] if image not in sampled: sampled[image] = [] pixel = index % self.pixels_per_image if image in self.positives_sorted and pixel in self.positives_sorted[image]: p = 1. positives += 1 else: p = 0. (x, y) = self.pixel_to_xy(pixel) sampled[image].append([x, y, p]) count += 1 if self.verbose: bar.update(count) self.sampled = sampled if positives > 0: TT.warn("> Out of", batch_size, "sampled pixels,", positives, "pixels are positive.") return sampled, count
def fit(self, X, y, M, batch_size=128, nb_epoch=100, verbose=1, validation_split=0., lr=None, shuffle=True): y = standardize_y(y) # If a validation split size is given (e.g. validation_split=0.2) # then split X into smaller X and X_val, # and split y into smaller y and y_val. do_validation = False if validation_split > 0 and validation_split < 1: do_validation = True split_at = int(len(X) * (1 - validation_split)) (X, X_val) = (X[0:split_at], X[split_at:]) (y, y_val) = (y[0:split_at], y[split_at:]) (M, M_val) = (M[0:split_at], M[split_at:]) if verbose: print "Train on %d samples, validate on %d samples" % (len(y), len(y_val)) index_array = numpy.arange(len(X)) for epoch in range(nb_epoch): if verbose: print 'Epoch', epoch if shuffle: numpy.random.shuffle(index_array) nb_batch = int(numpy.ceil(len(X)/float(batch_size))) progbar = Progbar(target=len(X)) for batch_index in range(0, nb_batch): batch_start = batch_index*batch_size batch_end = min(len(X), (batch_index+1)*batch_size) batch_ids = index_array[batch_start:batch_end] X_batch = X[batch_ids] y_batch = y[batch_ids] M_batch = M[batch_ids] loss = self._train(X_batch, y_batch, M_batch, lr) if verbose: is_last_batch = (batch_index == nb_batch - 1) if not is_last_batch or not do_validation: progbar.update(batch_end, [('loss', loss)]) else: progbar.update(batch_end, [('loss', loss), ('val. loss', self.test(X_val, y_val, M_val))])
# pdb.set_trace() # if not os.path.isfile(os.path.join(jsonpath,str(indtrain[i])+'.json')): # print('no file exist') # continue start = time.time() objvec = fget_activs(RGBX_train,OFX_train) model.fit({'objvec':objvec,'out':cur_label},batch_size=128,\ nb_epoch=1,shuffle=False,verbose=0,callbacks=[MPLC]) endfit = time.time() # pdb.set_trace() progbar.update(tidx+passed+1) reterr = model.predict({'objvec':objvec},verbose=0) batchtotAP = 0 reterr = reterr['out'] curAP, acc = getAP(reterr,cur_label) batchtotAP = batchtotAP + curAP totactAP = totactAP + batchtotAP lastacc = lastacc + acc curtotAP = float(totactAP) / (tidx+passed+1) endap = time.time()
def fit(self, X, Y, M, X_val, Y_val, M_val, lr_A=None, lr_B=None, batch_size=128, verbose=1, shuffle=True, epoch_start=0, continue_training=lambda *_: True, epoch_callback=lambda *_: None): # Require validation set to be explicitly passed in (to prevent risk # of dataset contamination with the embedding step) m = 10000 do_validation = False if X_val is not None and Y_val is not None and M_val is not None: do_validation = True if verbose: print "Train on {0} samples, validate on {1} samples".format(len(Y), len(Y_val)) print "epoch callback: {0}".format("yes" if epoch_callback else "no") index_array = numpy.arange(len(X)) epoch = epoch_start callback_result = None while continue_training(epoch, callback_result): if verbose: print 'Epoch', epoch if shuffle: numpy.random.shuffle(index_array) nb_batch = int(numpy.ceil(len(X)/float(batch_size))) progbar = Progbar(target=len(X)) for batch_index in range(0, nb_batch): batch_start = batch_index*batch_size batch_end = min(len(X), (batch_index+1)*batch_size) batch_ids = index_array[batch_start:batch_end] X_batch = X[batch_ids] Y_batch = Y[batch_ids] M_batch = M[batch_ids] loss = self._train(X_batch, Y_batch, M_batch, lr_A, lr_B) if verbose: is_last_batch = (batch_index == nb_batch - 1) if not is_last_batch or not do_validation: progbar.update(batch_end, [('loss', loss*m)]) else: #val_loss = self.test(X_val, Y_val, M_val) i_vt = random.sample(xrange(len(X_val)), batch_size) X_vt = numpy.array([X_val[i] for i in i_vt]) Y_vt = numpy.array([Y_val[i] for i in i_vt]) M_vt = numpy.array([M_val[i] for i in i_vt]) val_loss = self.test(X_vt, Y_vt, M_vt) progbar.update(batch_end, [('loss', loss*m), ('val. loss', val_loss*m)]) loss = progbar.sum_values['loss'][0] / (m * progbar.sum_values['loss'][1]) # call epoch callback after each round round_stats = {} round_stats['train'] = {'loss': float(loss)} if do_validation: round_stats['validate'] = {'loss': float(val_loss)} callback_result = epoch_callback(round_stats, epoch) epoch += 1
# y_train = readjson.to_categorical_dual(y_train,10) endread = time.time() if isGraph: model.fit({'input1':X_train,'actout':act_label,'objout':obj_label},batch_size=128,\ nb_epoch=1,shuffle=False,verbose=0,callbacks=[MPLC]) else: # print('model fit') model.fit(X_train,act_label,batch_size=128,nb_epoch=1,show_accuracy=True,shuffle=False,\ verbose=0,callbacks=[MPLC]) endfit = time.time() # pdb.set_trace() progbar.update(i) if showAP: x_each = X_train.transpose(1,0,2) if isGraph: reterr = model.predict({'input1':x_each},verbose=0) predobj = reterr['objout'] predact = reterr['actout'] batchobjAP = 0 batchactAP = 0 batchacc = 0 for batidx in range(len(predobj)): sortedobjerr = [si[0] for si in sorted(enumerate(predobj[batidx]),reverse=True,key=lambda xy:xy[1])] itemobjidx = np.where(obj_label[batidx]==1) # sortederr = [61,32,51,...] ( 0 ~ 154 ) # itemidx[0] = array([34,51,61]) ( 0 ~ 154 )
nb_train, nb_test = X_train.shape[0], X_test.shape[0] train_history = defaultdict(list) test_history = defaultdict(list) for epoch in range(nb_epochs): print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) nb_batches = int(X_train.shape[0] / batch_size) progress_bar = Progbar(target=nb_batches) epoch_gen_loss = [] epoch_disc_loss = [] for index in range(nb_batches): progress_bar.update(index) # generate a new batch of noise noise = np.random.uniform(-1, 1, (batch_size, latent_size)) # get a batch of real images image_batch = X_train[index * batch_size:(index + 1) * batch_size] label_batch = y_train[index * batch_size:(index + 1) * batch_size] # sample some labels from p_c sampled_labels = np.random.randint(0, 10, batch_size) # generate a batch of fake images, using the generated labels as a # conditioner. We reshape the sampled labels to be # (batch_size, 1) so that we can feed them into the embedding # layer as a length one sequence generated_images = generator.predict(
def run_training(trainfile, testfile, embeddings_file, epochs, static=False, maxlen=100, batch_size=32): print('Loading data...') sents_train, truths_train, unique_words_train, unique_tags_train = \ P.retrieve_sentences_tags(trainfile, maxlen=maxlen) sents_test, truths_test, unique_word_test, unique_tags_test = \ P.retrieve_sentences_tags(testfile, maxlen=maxlen, allowedtags=unique_tags_train) alltags = unique_tags_train.union(unique_tags_test) uniqueWords = unique_words_train.union(unique_word_test) gsm_mod = gensim.models.Word2Vec.load_word2vec_format(embeddings_file) vocab_dim = len(gsm_mod['word']) tagDict = {} for n, t in enumerate(alltags): tagDict[t] = n + 1 index_dict = {} for n, word in enumerate(uniqueWords): index_dict[word] = n + 1 nb_classes = len(tagDict) X_train, Y_train = P.create_input_data(sents_train, truths_train, index_dict, tagDict, maxlen=maxlen) X_test, Y_test = P.create_input_data(sents_test, truths_test, index_dict, tagDict, maxlen=maxlen) # makes output classes binary vectors instead of class numbers Y_train_cat = np.array([to_categorical(y, nb_classes=nb_classes + 1) for y in Y_train]) Y_test_cat = np.array([to_categorical(y, nb_classes=nb_classes + 1) for y in Y_test]) print(Y_train_cat.shape) print(X_train.shape) n_symbols = len(uniqueWords) + 1 # adding 1 to account for 0th index (for masking) embedding_weights = np.zeros((n_symbols, vocab_dim)) for word, index in index_dict.items(): embedding_weights[index, :] = gsm_mod[word] # assemble the model model = Sequential() if not static: model.add(Embedding(output_dim=vocab_dim, input_dim=n_symbols, mask_zero=False, weights=[embedding_weights])) model.add(LSTM(128, return_sequences=True, input_shape=(maxlen, vocab_dim))) model.add(Dropout(0.5)) model.add(TimeDistributed(Dense(nb_classes + 1))) model.add(Activation('softmax')) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) keep_iterating = True count = 0 cwd = os.getcwd() while keep_iterating: # making sure to not save the weights as the same name as # another is using count += 1 tmpweights = "{}/tmp/weights{}.hdf5".format(cwd, count) if not os.path.isfile(tmpweights): keep_iterating = False print('============Training Params============\n' 'Training file: {}\nTesting file: {}\nEmbeddings file: {}\n' 'Epochs: {}\nStatic: {}\nWord embedding dimensions: {}\n' 'Batch size: {}\nMax length of sentence: {}\n' '=======================================' .format(trainfile, testfile, embeddings_file, epochs, static, vocab_dim, batch_size, maxlen)) print('Train...') best_yet = 0 accs = [] for e in range(epochs): print("Training epoch {}".format(e + 1)) pbar = Progbar(1 + len(X_train) / batch_size) batch_count = 0 for xt, yt in batch(X_train, Y_train_cat, vocab_dim, embedding_weights, static, n=batch_size, shuffle=True): batch_count += 1 model.fit(xt, yt, batch_size=batch_size, nb_epoch=1, verbose=False) pbar.update(batch_count) # free up some space? maybe python automatically garbage collects already xt = None yt = None validation_size = 1024 print("Training finished, evaluating on {} validation samples".format(validation_size)) # take a random subset of validation data for X_test_subset, Y_test_subset in batch(X_test, Y_test, vocab_dim, embedding_weights, static, n=validation_size, shuffle=True): hypo = model.predict_classes(X_test_subset, batch_size=1) break correct, incorrect = custom_accuracy(y_true=Y_test_subset, y_pred=hypo) acc = correct / float(correct + incorrect) accs.append(acc) print("Correct: {}\nIncorrect: {}\n Accuracy: {}" .format(correct, incorrect, acc)) if acc > best_yet: print('Improved from {} to {}, saving weights to {}\nEpoch {} finished.' .format(best_yet, acc, tmpweights, e + 1)) best_yet = acc model.save_weights(tmpweights, overwrite=True) model.load_weights(tmpweights) # evaluate on model's best weights first = True for xt, yt in batch(X_test, Y_test_cat, vocab_dim, embedding_weights, static, n=validation_size): hypo = model.predict_classes(xt, batch_size=1) if first: Y_hypo = hypo first = False else: Y_hypo = np.concatenate((Y_hypo, hypo)) correct, incorrect = custom_accuracy(y_true=Y_test, y_pred=Y_hypo) print("Finished! Final Score\nCorrect: {}\nIncorrect: {}\n Accuracy: {}" .format(correct, incorrect, float(correct) / (correct + incorrect))) log = '{}/tmp/log_{}.txt'.format(cwd, count) f = open(log, 'w') f.write('Embeddings file: {}\n'.format(embeddings_file)) f.write('Accuracy for each epoch: {}\n'.format(str(accs))) f.close() print('Log saved as {}'.format(log))
losses = [] batch_loss = [] for i, seq in enumerate(tokenizer.texts_to_sequences_generator(text_generator())): # get skipgram couples for one text in the dataset couples, labels = skipgrams_l2c_fast(seq, vocab_size, num_senses =num_senses, window_size=4, negative_samples=1., sampling_table=sampling_table) if couples: # one gradient update per sentence (one sentence = a few 1000s of word couples) # print couples X = np.array(couples, dtype="int32") labels= np.array(labels, dtype="int32") loss = model.train_on_batch(X, labels) losses.append(loss) batch_loss.append(loss) if len(losses) % 10 == 0: print ('\nBatch Loss: '+str(np.mean(batch_loss))) progbar.update(i, values=[("loss", np.mean(losses))]) batch_loss = [] samples_seen += len(labels) if (i and i % 10000 == 0): global_weights, sense_weights = model.layers[0].get_weights()[0] avgSim, avgSimC = evaluator.get_scores(global_weights, sense_weights) print("scores after %d epochs:") print("\t avg-sim: %5.3f", avgSim) print("\t global-sim: %5.3f", avgSimC) print('Samples seen:', samples_seen)
# Train ConvNet from keras.utils.generic_utils import Progbar batch_losses = np.zeros(epoch_size / batch_size) loss_history = [] mean_loss = float("inf") for epoch_id in xrange(n_epochs): dataflow = datagen.flow(batch_size=batch_size, epoch_size=epoch_size) print "\nEpoch ", 1 + epoch_id progbar = Progbar(epoch_size) batch_id = 0 for (X_batch, Y_batch) in dataflow: loss = di.learning.train_on_batch(graph, X_batch, Y_batch, Q, js, offsets) batch_losses[batch_id] = loss[0] progbar.update(batch_id * batch_size) batch_id += 1 if np.mean(batch_losses) < mean_loss: mean_loss = np.mean(batch_losses) std_loss = np.std(batch_losses) else: break print "\nTraining loss = ", mean_loss, " +/- ", std_loss # Measure test accuracies class_probs = di.learning.predict(graph, X_test, Q, js, offsets) y_predicted = np.argmax(class_probs, axis=1) chunk_accuracies = di.singlelabel.chunk_accuracies(y_predicted, y_test) file_accuracies = di.singlelabel.file_accuracies(test_paths, class_probs, y_test, method="geometric_mean") mean_file_accuracy = np.mean(file_accuracies)
def update_general(self, X, y, w, load_func, reset=False): log = logging.getLogger(__name__) data_size = y.shape[0] #figure out the weights if w is None and len(y.shape)==1: y_unique = np.unique(y) weights = class_weight.compute_class_weight(self.class_weight, y_unique, y) self.class_actual_weight_ = {} for i, y_val in enumerate(y_unique): self.class_actual_weight_[y_val] = weights[i] w = np.zeros(data_size) for i,v in enumerate(y): w[i] = self.class_actual_weight_[v] elif w is None: log.warn('Do not know how to make class weights for multidimensinal output. If neeed, specify weights directly. Assuming uniform weights.') w = np.ones(data_size) else: assert w.shape[0]==data_size, 'Weight size should match data size.' if self.background: queue = multiprocessing.Queue() else: queue = Queue.Queue() log.info('Starting to fit the NN model.') if self.callbacks is not None: for callback in self.callbacks: if callback is not None: callback.on_train_begin(self) for epoch in xrange(self.nb_epoch): last_update = time.time()-1000 start_time = time.time() #generate the progress bar if self.verbose>0: progbar = Progbar(data_size, width=80, verbose=self.verbose) #get random permutation p = np.random.permutation(range(data_size)) #load the first batch batch_idx = p[0:self.memory_batch_size]; self.matrix_load_into_queue(X, batch_idx, queue, load_func, y, w); X_batch,y_batch,w_batch = queue.get() if reset and epoch==0: n_features = self.get_dimensions(X_batch) log.info('Compiling the NN model with {} dimensions.'.format(n_features)) self.generate_and_compile_model_(n_features) samples = 0 for batch, i in enumerate(xrange(0, len(p), self.memory_batch_size)): #compute indicies for next batch next_start = i+len(batch_idx) next_end = min(len(p), next_start+self.memory_batch_size) if next_end>next_start: #spin the thread up batch_idx_next = p[next_start:next_end]; #load data in background thread = 0 if self.background: thread = multiprocessing.Process(target=self.matrix_load_into_queue, args=(X,batch_idx_next,queue,load_func, y, w)) thread.start() else: batch_idx_next = None thread = None #perform update loss = self.batch_update(X_batch, y_batch, w_batch) #increment the counter samples+= len(batch_idx) curr_update = time.time() if self.verbose>0 and (curr_update-last_update>=0.5 or (samples)>=len(p)): progbar.update(samples, [('Loss', loss)]) last_update = curr_update if self.callbacks is not None: for callback in self.callbacks: if callback is not None: r = callback.on_batch_end(self, epoch+1, batch+1) #wait for the next load to happen if thread is not None: #if no background, load the data now if not self.background: self.matrix_load_into_queue(X, batch_idx, queue, load_func, y, w) X_batch,y_batch,w_batch = queue.get() #if loading a background process, do a join if self.background: thread.join() #now add the next batch batch_idx = batch_idx_next finish_time = time.time()-start_time if self.verbose>0: log.info('Finished epoch {}/{}. Time per epoch (s): {:0.2f}, Time per sample (s): {}.'.format(epoch+1, self.nb_epoch,finish_time,finish_time/len(p))) #process the end of epoch, and see if need to quit out quit_now = False if self.callbacks is not None: for callback in self.callbacks: if callback is not None: r = callback.on_epoch_end(self, epoch+1) if r is not None and r is True: quit_now = True if quit_now: break return self
class LangModelLogger(BaseLogger): def __init__(self): super(LangModelLogger, self).__init__() self.verbose = None self.nb_epoch = None self.seen = 0 self.totals = {} self.progbar = None self.log_values = [] # def on_train_begin(self, logs=None): # logger.debug('Begin training...') # self.verbose = self.params['verbose'] # self.nb_epoch = self.params['nb_epoch'] # # def on_epoch_begin(self, epoch, logs=None): # # print('Epoch %d/%d' % (epoch + 1, self.nb_epoch)) # self.progbar = Progbar(target=self.params['nb_sample'], verbose=1) # self.seen = 0 # self.totals = {} # # def on_batch_begin(self, batch, logs=None): # if self.seen < self.params['nb_sample']: # self.log_values = [] # self.params['metrics'] = ['loss', 'ppl', 'val_loss', 'val_ppl'] def on_batch_end(self, batch, logs=None): logs = {} if logs is None else logs batch_size = logs.get('size', 0) self.seen += batch_size for k, v in logs.items(): if k == 'encode_len' or 'nb_words': try: self.totals[k] += v except KeyError: self.totals[k] = v continue try: self.totals[k] += v * batch_size except KeyError: self.totals[k] = v * batch_size if 'encode_len' in self.totals and 'nb_words' in self.totals and 'ppl' in self.params['metrics']: self.totals['ppl'] = math.exp(self.totals['encode_len']/float(self.totals['nb_words'])) self.log_values.append(('ppl', self.totals['ppl'])) for k in self.params['metrics']: if k in logs: self.log_values.append((k, logs[k])) # skip progbar update for the last batch; will be handled by on_epoch_end if self.seen < self.params['nb_sample']: self.progbar.update(self.seen, self.log_values) def on_epoch_begin(self, epoch, logs=None): if self.verbose: self.progbar = Progbar(target=self.params['nb_sample'], verbose=self.verbose) self.seen = 0 self.totals = {} def on_epoch_end(self, epoch, logs=None): logs = {} if logs is None else logs # logger.debug('log keys: %s' % str(logs.keys())) for k in self.params['metrics']: if k in self.totals: if k != 'ppl': self.log_values.append((k, self.totals[k] / self.seen)) else: self.totals['ppl'] = math.exp(self.totals['encode_len']/float(self.totals['nb_words'])) self.log_values.append((k, self.totals['ppl'])) if k in logs: self.log_values.append((k, logs[k])) if 'val_encode_len' in logs and 'val_nb_words' in logs: val_ppl = math.exp(logs['val_encode_len']/float(logs['val_nb_words'])) self.log_values.append(('val_ppl', val_ppl)) self.progbar.update(self.seen, self.log_values)