def start_training(config, profile_step): use_horovod = horovod_util.is_enabled() print("use_horovod:", use_horovod) if use_horovod: hvd = horovod_util.setup() rank = hvd.rank() local_rank = hvd.local_rank() else: rank = 0 local_rank = -1 ModelClass = config.NETWORK_CLASS network_kwargs = {key.lower(): val for key, val in config.NETWORK.items()} train_dataset = setup_dataset(config, "train", rank, local_rank) print("train dataset num:", train_dataset.num_per_epoch) validation_dataset = setup_dataset(config, "validation", rank, local_rank) print("validation dataset num:", validation_dataset.num_per_epoch) graph = tf.Graph() with graph.as_default(): if config.TASK == Tasks.OBJECT_DETECTION: model = ModelClass( classes=train_dataset.classes, num_max_boxes=train_dataset.num_max_boxes, is_debug=config.IS_DEBUG, **network_kwargs, ) else: model = ModelClass( classes=train_dataset.classes, is_debug=config.IS_DEBUG, **network_kwargs, ) is_training_placeholder = tf.compat.v1.placeholder( tf.bool, name="is_training_placeholder") images_placeholder, labels_placeholder = model.placeholders() output = model.inference(images_placeholder, is_training_placeholder) loss = model.loss(output, labels_placeholder) opt = model.optimizer() if use_horovod: # add Horovod Distributed Optimizer opt = hvd.DistributedOptimizer(opt) train_op = model.train(loss, opt) metrics_ops_dict, metrics_update_op = model.metrics( output, labels_placeholder) # TODO(wakisaka): Deal with many networks. model.summary(output, labels_placeholder) summary_op = tf.compat.v1.summary.merge_all() metrics_summary_op = executor.metrics_summary_op(metrics_ops_dict) init_op = tf.compat.v1.global_variables_initializer() reset_metrics_op = tf.compat.v1.local_variables_initializer() if use_horovod: # add Horovod broadcasting variables from rank 0 to all bcast_global_variables_op = hvd.broadcast_global_variables(0) saver = tf.compat.v1.train.Saver( max_to_keep=config.KEEP_CHECKPOINT_MAX) with file_io.File(os.path.join(environment.EXPERIMENT_DIR, "pretrain_vars.txt"), mode="w") as f: train_vars = tf.compat.v1.trainable_variables() f.write("[\n") f.write("".join(" '%s',\n" % var.name for var in train_vars)) f.write("]\n") if config.IS_PRETRAIN: all_vars = tf.compat.v1.global_variables() pretrain_var_list = [ var for var in all_vars if var.name.startswith(tuple(config.PRETRAIN_VARS)) ] print("pretrain_vars", [var.name for var in pretrain_var_list]) pretrain_saver = tf.compat.v1.train.Saver(pretrain_var_list, name="pretrain_saver") if use_horovod: # For distributed training session_config = tf.compat.v1.ConfigProto( gpu_options=tf.compat.v1.GPUOptions( allow_growth=True, visible_device_list=str(hvd.local_rank()))) else: # TODO(wakisaka): For debug. # session_config = tf.ConfigProto( # gpu_options=tf.GPUOptions( # allow_growth=True, # per_process_gpu_memory_fraction=0.1 # ) # ) session_config = tf.compat.v1.ConfigProto( ) # tf.ConfigProto(log_device_placement=True) # TODO(wakisaka): XLA JIT # session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 sess = tf.compat.v1.Session(graph=graph, config=session_config) sess.run([init_op, reset_metrics_op]) executor.save_pb_file(sess, environment.CHECKPOINTS_DIR) if rank == 0: train_writer = tf.compat.v1.summary.FileWriter( environment.TENSORBOARD_DIR + "/train", sess.graph) val_writer = tf.compat.v1.summary.FileWriter( environment.TENSORBOARD_DIR + "/validation") if config.IS_PRETRAIN: print("------- Load pretrain data ----------") pretrain_saver.restore( sess, os.path.join(config.PRETRAIN_DIR, config.PRETRAIN_FILE)) # for recovery ckpt = tf.train.get_checkpoint_state(environment.CHECKPOINTS_DIR) if ckpt and ckpt.model_checkpoint_path: print("--------- Restore last checkpoint -------------") saver.restore(sess, ckpt.model_checkpoint_path) # saver.recover_last_checkpoints(ckpt.model_checkpoint_path) last_step = sess.run(model.global_step) # TODO(wakisaka): tensorflow v1.3 remain previous event log in tensorboard. # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/python/training/supervisor.py#L1072 train_writer.add_session_log(SessionLog(status=SessionLog.START), global_step=last_step + 1) val_writer.add_session_log(SessionLog(status=SessionLog.START), global_step=last_step + 1) print("recovered. last step", last_step) if use_horovod: # broadcast variables from rank 0 to all other processes sess.run(bcast_global_variables_op) last_step = sess.run(model.global_step) # Calculate max steps. The priority of config.MAX_EPOCHS is higher than config.MAX_STEPS. if "MAX_EPOCHS" in config: max_steps = int(train_dataset.num_per_epoch / config.BATCH_SIZE * config.MAX_EPOCHS) if max_steps < 1: print( "The max_steps is less than 1, consider reduce BATCH_SIZE. exit.", file=sys.stderr) sys.exit(1) else: max_steps = config.MAX_STEPS if max_steps < 1: print( "The max_steps is less than 1, consider set MAX_STEPS greater than 0. exit.", file=sys.stderr) sys.exit(1) progbar = Progbar(max_steps) if rank == 0: progbar.update(last_step) for step in range(last_step, max_steps): images, labels = train_dataset.feed() feed_dict = { is_training_placeholder: True, images_placeholder: images, labels_placeholder: labels, } # Runtime statistics for develop. if step == profile_step: options = tf.compat.v1.RunOptions( trace_level=tf.compat.v1.RunOptions.FULL_TRACE) run_meta = tf.compat.v1.RunMetadata() else: options = None run_meta = None if step * ((step + 1) % config.SUMMARISE_STEPS) == 0 and rank == 0: sess.run(reset_metrics_op) _, summary, _ = sess.run( [train_op, summary_op, metrics_update_op], feed_dict=feed_dict, options=options, run_metadata=run_meta, ) # train_writer.add_run_metadata(run_metadata, "step: {}".format(step + 1)) train_writer.add_summary(summary, step + 1) metrics_summary = sess.run(metrics_summary_op) train_writer.add_summary(metrics_summary, step + 1) train_writer.flush() else: sess.run([train_op], feed_dict=feed_dict, options=options, run_metadata=run_meta) if step == profile_step: executor.profile_train_step(step, sess, run_meta) to_be_saved = step == 0 or ( step + 1) == max_steps or (step + 1) % config.SAVE_CHECKPOINT_STEPS == 0 if to_be_saved and rank == 0: _save_checkpoint(saver, sess, model.global_step) if step == 0 or (step + 1) % config.TEST_STEPS == 0: # init metrics values sess.run(reset_metrics_op) test_step_size = int( math.ceil(validation_dataset.num_per_epoch / config.BATCH_SIZE)) for test_step in range(test_step_size): images, labels = validation_dataset.feed() feed_dict = { is_training_placeholder: False, images_placeholder: images, labels_placeholder: labels, } if test_step % config.SUMMARISE_STEPS == 0: summary, _ = sess.run([summary_op, metrics_update_op], feed_dict=feed_dict) if rank == 0: val_writer.add_summary(summary, step + 1) val_writer.flush() else: sess.run([metrics_update_op], feed_dict=feed_dict) metrics_summary = sess.run(metrics_summary_op) if rank == 0: val_writer.add_summary(metrics_summary, step + 1) val_writer.flush() if rank == 0: progbar.update(step + 1) # training loop end. train_dataset.close() validation_dataset.close() print("Done")
test_accuracy = tf.keras.metrics.CategoricalAccuracy() test_loss = tf.keras.metrics.CategoricalCrossentropy(from_logits=True) from tensorflow.keras.utils import Progbar model = model_mnist(0) # !!! source = [] for i in range(4): # !!! source.append( tf.keras.models.load_model('./Models/ens/model_' + str(i) + '.h5')) for epoch in range(num_epochs): train_dataset = get_training_dataset() print("\nepoch {}/{}".format(epoch + 1, num_epochs)) pb_i = Progbar(x_train.shape[0], stateful_metrics=['loss', 'acc'], verbose=2) for x, y in train_dataset: i = np.random.choice([0, 2, 3]) # !!! # PGD Adversarial Training # adv_train(model,x,y,adv_params='pgd_76_10_8',random_init=True) #!!! # Ensemble Adversarial Training ## x_adv = generate_adversarial_examples(x, bounds, source[i], 'pgd_76_76_1', random_init=False) ## normal_train(model,x,y) # ens_train(source[i],model,x,y,adv_params='pgd_76_76_1',random_init=False) #!!! # Projection Regularization pr_train(source[i], model, x, y, alpha=0.3, beta=10, sigma=sigma) # !!!
def eval_results(ds_name, encoder, paths): """The main function for executing network training. It loads the specified dataset iterator, saliency model, and helper classes. Training is then performed in a new session by iterating over all batches for a number of epochs. After validation on an independent set, the model is saved and the training history is updated. Args: ds_name (str): Denotes the dataset to be used during training. paths (dict, str): A dictionary with all path elements. """ w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5 (eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"]) print(">> Preparing model with encoder %s..." % encoder) model = MyModel(encoder, ds_name, "train") if "trained_weights" in paths: if os.path.exists(paths["trained_weights"]): weights_path = paths["trained_weights"] else: raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"]) else: weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name) if os.path.exists(weights_path): print("Weights are loaded!\n %s"%weights_path) else: download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name) model.load_weights(weights_path) del weights_path model.summary() # Preparing metrics = config.PARAMS["metrics"] print("\n>> Start evaluating model on %s..." % ds_name.upper()) print(("Evaluation details:" + "\n{0:<4}Metrics: {2}").format(" ", loss_fn_name, ", ".join(metrics), **config.PARAMS)) print("_" * 65) eval_progbar = Progbar(n_eval) categorical = config.SPECS[ds_name].get("categorical", False) cat_metrics = {} for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds: eval_y_pred = test_step(eval_x, model) for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes): pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0) fixs = tf.expand_dims(fixs, axis=0) y_true = tf.expand_dims(y_true, axis=0) met_vals = _calc_metrics(metrics, y_true, fixs, pred) if categorical: cat = "/".join(filename.decode("utf-8").split("/")[:-1]) if not cat in cat_metrics: cat_metrics[cat] = {} for name in metrics: cat_metrics[cat][name] = {"sum":0, "count": 0} for name, value in met_vals.items(): cat_metrics[cat][name]["sum"] += value cat_metrics[cat][name]["count"] += 1 eval_progbar.add(eval_x.shape[0], met_vals.items()) for cat, cat_met in cat_metrics.items(): to_print = [] for name, value in cat_met.items(): _mean = value["sum"]/value["count"] to_print.append("{}: {}".format(name, ('%.4f' if _mean > 1e-3 else '%.4e') % _mean)) print('Results ({}): {}'.format(cat, " - ".join(to_print)))
def validation_loop(sess, g, n_batches, chars=None, val_gen=None, tb_writer=None): Loss = [] Cer = [] Wer = [] Predictions = [] progbar = Progbar(target=n_batches, verbose=1, stateful_metrics=['t']) print('Starting validation Loop') for i in range(n_batches): x, y = val_gen.next() if len(x) == 1: x = x[0] if len(y) == 1: y = y[0] # -- Autoregressive inference preds = np.zeros((config.batch_size, config.maxlen), np.int32) tile_preds = config.test_aug_times # -- For train graph feed in the previous step's predictions manually for the next if not 'infer' in config.graph_type: prev_inp = np.tile( preds, [config.test_aug_times, 1]) if tile_preds else preds feed_dict = {g.x: x, g.prev: prev_inp, g.y: y} enc = sess.run(g.enc, feed_dict) if type(enc) is list: for enc_tens, enc_val in zip(g.enc, enc): feed_dict[enc_tens] = enc_val else: feed_dict[g.enc] = enc for j in range(config.maxlen): _preds, loss, cer = sess.run([g.preds, g.mean_loss, g.cer], feed_dict) preds[:, j] = _preds[:, j] prev_inp = np.tile( preds, [config.test_aug_times, 1]) if tile_preds else preds feed_dict[g.prev] = prev_inp # if all samples in batch predict the pad symbol (char_id==0) if np.sign(preds[:, j]).sum() == 0: if g.tb_sum is not None: tb_sum = sess.run(g.tb_sum, { g.x: x, g.prev: prev_inp, g.y: y }) break # -- Autoregression loop is built into the beam search graph else: feed_dict = {g.x: x, g.y: y} enc = sess.run(g.enc, feed_dict) if type(enc) is list: for enc_tens, enc_val in zip(g.enc, enc): feed_dict[enc_tens] = enc_val else: feed_dict[g.enc] = enc _preds, loss, cer = sess.run([g.preds, g.mean_loss, g.cer], feed_dict) preds = _preds # use last loss gt_sents = [''.join([chars[cid] for cid in prr]).strip() for prr in y] gt_words = [sent.split('-') for sent in gt_sents] def decode_preds_to_chars(decoding): return ''.join([chars[cid] for cid in decoding]).strip() pred_sentences = [decode_preds_to_chars(prr) for prr in preds] pred_words = [sent.split('-') for sent in pred_sentences] edists = [ rel_edist(gt, dec_str) for gt, dec_str in zip(gt_words, pred_words) ] wer = np.mean(edists) # -- Write tb_summaries if any if g.tb_sum is not None: if wer == 0: tb_writer.add_summary(tb_sum, i) if config.print_predictions: print() for gts, prs, wr in zip(gt_sents, pred_sentences, edists): print('(wer={:.1f}) {} --> {}'.format(wr * 100, gts, prs)) progbar.update(i + 1, [('cer', cer), ('wer', wer)]) Wer.append(wer) Predictions.append(pred_sentences) Cer.append(cer) Loss.append(loss) return np.average(Loss), np.average(Cer), np.average(Wer), Predictions[0]
def main(_): with open(os.path.join(FLAGS.configuration_dir, "config.json")) as f: configuration = json.load(f) add_conf_dir_to_paths(configuration, FLAGS.configuration_dir) vocab_file = configuration["general"]["vocab_file"] input_data_file = configuration["pretraining"]["data-generation"][ "input_data_file"] num_shards = configuration["pretraining"]["data-generation"]["num_shards"] random_seed = configuration["pretraining"]["data-generation"][ "random_seed"] max_seq_length = configuration["general"]["max_seq_length"] dupe_factor = configuration["pretraining"]["data-generation"][ "dupe_factor"] max_predictions_per_seq = configuration["pretraining"]["data-generation"][ "max_predictions_per_seq"] masked_lm_prob = configuration["pretraining"]["data-generation"][ "masked_lm_prob"] output_data_dir = configuration["pretraining"]["data-generation"][ "output_data_dir"] bert_config_file = configuration["general"]["bert_config_file"] bert_config = modeling.BertConfig.from_json_file(bert_config_file) tf.logging.set_verbosity(tf.logging.INFO) tokenizer = tokenization.SpaceTokenizer(vocab=vocab_file, vocab_size=bert_config.vocab_size) input_files = [input_data_file] output_files = [ os.path.join(output_data_dir, "train_{0:04d}.tfrecord".format(i)) for i in range(num_shards) ] tf.logging.info("*** Reading from input files ***") for input_file in input_files: tf.logging.info(" %s", input_file) tf.logging.info("*** Writing to output files ***") for output_file in output_files: tf.logging.info(" %s", output_file) writers = [] for output_file in output_files: writers.append(tf.python_io.TFRecordWriter(output_file)) rng = random.Random(random_seed) bar = Progbar(num_shards) instances_written = 0 all_examples = [] for i in range(num_shards): bar.add(1) examples, instances = create_training_instances( (i, num_shards), input_files, writers, tokenizer, max_seq_length, dupe_factor, masked_lm_prob, max_predictions_per_seq, rng) instances_written += instances all_examples.extend(examples) rng.shuffle(all_examples) for example in all_examples[:FLAGS.num_show_examples]: write_instance_to_example_files(example, None, tokenizer, max_seq_length, max_predictions_per_seq, True) tf.logging.info("Wrote %d instances.\n", instances_written) for writer in writers: writer.close()
def train(training_data, validation_data, cfg): x_train, y_train, y_train_cat = training_data x_test, y_test, y_test_cat = validation_data test_indices = np.arange(len(x_test)) # get the training data generator. We are not using validation generator because the # data is already loaded in memory and we don't have to perform any extra operation # apart from loading the validation images and validation labels. ds = DataGenerator(x_train, y_train_cat, batch_size=cfg.batch_size) enqueuer = OrderedEnqueuer(ds, use_multiprocessing=True) enqueuer.start(workers=multiprocessing.cpu_count()) train_ds = enqueuer.get() # get the total number of training and validation steps nb_train_steps = int(np.ceil(len(x_train) / cfg.batch_size)) nb_test_steps = int(np.ceil(len(x_test) / cfg.batch_size)) global total_steps total_steps = nb_train_steps * cfg.num_epochs # get the optimizer # SGD with cosine lr is causing NaNs. Need to investigate more optim = optimizers.Adam(learning_rate=0.001) # checkpoint prefix checkpoint_prefix = os.path.join(cfg.save_dir_path, "ckpt") checkpoint = tf.train.Checkpoint(optimizer=optim, model=model) checkpoint_manager = tf.train.CheckpointManager( checkpoint, directory=cfg.save_dir_path, max_to_keep=10) # check for previous checkpoints, if any checkpoint.restore(checkpoint_manager.latest_checkpoint) if checkpoint_manager.latest_checkpoint: print("Checkpoint restored from {}".format( checkpoint_manager.latest_checkpoint)) starting_epoch = checkpoint.save_counter.numpy() else: print("Initializing from scratch.") starting_epoch = 0 # sanity check for epoch number. For example, if someone restored a checkpoint # from 15th epoch and want to train for two more epochs, then we need to explicitly # encode this logic in the for loop if cfg.num_epochs <= starting_epoch: cfg.num_epochs += starting_epoch for epoch in range(starting_epoch, cfg.num_epochs): pbar = Progbar(target=nb_train_steps, interval=0.5, width=30) # Train for an epoch and keep track of # loss and accracy for each batch. for bno, (images, labels) in enumerate(train_ds): if bno == nb_train_steps: break # Get the batch data clean, aug1, aug2 = images loss_value, y_pred_clean = train_step(clean, aug1, aug2, labels, optim) # Record batch loss and batch accuracy train_loss(loss_value) train_accuracy(labels, y_pred_clean) pbar.update(bno + 1) # Validate after each epoch for bno in range(nb_test_steps): # Get the indices for the current batch indices = test_indices[bno * cfg.batch_size:(bno + 1) * cfg.batch_size] # Get the data images, labels = x_test[indices], y_test_cat[indices] # Get the predicitions and loss for this batch loss_value, y_pred = validate_step(images, labels) # Record batch loss and accuracy test_loss(loss_value) test_accuracy(labels, y_pred) # get training and validataion stats # after one epoch is completed loss = train_loss.result() acc = train_accuracy.result() val_loss = test_loss.result() val_acc = test_accuracy.result() # record values in the history object history.update([loss, acc], [val_loss, val_acc]) # print loss values and accuracy values for each epoch # for both training as well as validation sets print(f"""Epoch: {epoch+1} train_loss: {loss:.6f} train_acc: {acc*100:.2f}% test_loss: {val_loss:.6f} test_acc: {val_acc*100:.2f}%\n""") ## wandb logging wandb.log({ 'Epoch': epoch, 'train loss': loss, 'train acc': acc, 'test loss': val_loss, 'test acc': val_acc }) # get the model progress improved, stop_training = es.check_progress(val_loss) # check if performance of model has imporved or not if improved: print("Saving model checkpoint.") checkpoint.save(checkpoint_prefix) if stop_training: break # plot and save progression history.plot_and_save(initial_epoch=starting_epoch) # Reset the losses and accuracy train_loss.reset_states() train_accuracy.reset_states() test_loss.reset_states() test_accuracy.reset_states() print("") print("*" * 78) print("")
def load_weights_from_tf_checkpoint(model, checkpoint_file, background_label): print('Load weights from tensorflow checkpoint') progbar = Progbar(target=len(model.layers)) reader = tf.compat.v1.train.NewCheckpointReader(checkpoint_file) # pprint.pprint(reader.debug_string().decode("utf-8")) #类型是str for index, layer in enumerate(model.layers): progbar.update(current=index) print("layer.name", layer.name) print("layer", layer) if isinstance(layers, MobileNetV3Extractor): for index, llayers in enumerate(layers.layers): progbar.update(current=index) print("llayers.name", llayers.name) if isinstance(llayers, layers.SeparableConv2D): depthwise = reader.get_tensor( '{}/depthwise_weights'.format(llayers.name)) pointwise = reader.get_tensor( '{}/pointwise_weights'.format(llayers.name)) if K.image_data_format() == 'channels_first': depthwise = convert_kernel(depthwise) pointwise = convert_kernel(pointwise) llayers.set_weights([depthwise, pointwise]) elif isinstance(llayers, layers.Conv2D): weights = reader.get_tensor('{}/weights'.format( llayers.name)) if K.image_data_format() == 'channels_first': weights = convert_kernel(weights) llayers.set_weights([weights]) elif isinstance(llayers, layers.BatchNormalization): beta = reader.get_tensor('{}/beta'.format(llayers.name)) gamma = reader.get_tensor('{}/gamma'.format(llayers.name)) moving_mean = reader.get_tensor('{}/moving_mean'.format( llayers.name)) moving_variance = reader.get_tensor( '{}/moving_variance'.format(llayers.name)) llayers.set_weights( [gamma, beta, moving_mean, moving_variance]) elif isinstance(llayers, layers.Dense): weights = reader.get_tensor('{}/weights'.format( llayers.name)) biases = reader.get_tensor('{}/biases'.format( llayers.name)) if background_label: llayers.set_weights([weights, biases]) else: llayers.set_weights([weights[:, 1:], biases[1:]]) elif isinstance(layer, layers.SeparableConv2D): depthwise = reader.get_tensor('{}/depthwise_weights'.format( layer.name)) pointwise = reader.get_tensor('{}/pointwise_weights'.format( layer.name)) if K.image_data_format() == 'channels_first': depthwise = convert_kernel(depthwise) pointwise = convert_kernel(pointwise) layer.set_weights([depthwise, pointwise]) elif isinstance(layer, layers.Conv2D): weights = reader.get_tensor('{}/weights'.format(layer.name)) if K.image_data_format() == 'channels_first': weights = convert_kernel(weights) layer.set_weights([weights]) elif isinstance(layer, layers.BatchNormalization): beta = reader.get_tensor('{}/beta'.format(layer.name)) gamma = reader.get_tensor('{}/gamma'.format(layer.name)) moving_mean = reader.get_tensor('{}/moving_mean'.format( layer.name)) moving_variance = reader.get_tensor('{}/moving_variance'.format( layer.name)) layer.set_weights([gamma, beta, moving_mean, moving_variance]) elif isinstance(layer, layers.Dense): weights = reader.get_tensor('{}/weights'.format(layer.name)) biases = reader.get_tensor('{}/biases'.format(layer.name)) if background_label: layer.set_weights([weights, biases]) else: layer.set_weights([weights[:, 1:], biases[1:]])
def represent(self, molecules): """ provides bag of bonds representation for input molecules. Parameters ---------- molecules: chemml.chem.Molecule object or array If list, it must be a list of chemml.chem.Molecule objects, otherwise we raise a ValueError. In addition, all the molecule objects must provide the XYZ information. Please make sure the XYZ geometry has been stored or optimized in advance. Returns ------- features: pandas data frame, shape: (n_molecules, max_length_of_combinations) The bag of bond features. """ if isinstance(molecules, (list, np.ndarray)): molecules = np.array(molecules) elif isinstance(molecules, Molecule): molecules = np.array([molecules]) else: msg = "The input molecules must be a chemml.chem.Molecule object or a list of objects." raise ValueError(msg) if molecules.ndim > 1: msg = "The molecule must be a chemml.chem.Molecule object or a list of objects." raise ValueError(msg) # pool of processes if self.n_jobs == -1: self.n_jobs = cpu_count() pool = Pool(processes=self.n_jobs) # Create an iterator # http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks def chunks(l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] # find size of each batch batch_size = int(len(molecules) / self.n_jobs) if batch_size == 0: batch_size = 1 molecule_chunks = chunks(molecules, batch_size) # MAP: CM in parallel map_function = partial(self._represent) if self.verbose: print('featurizing molecules in batches of %i ...' % batch_size) pbar = Progbar(len(molecules), width=50) bbs_info = [] for tensors in pool.imap(map_function, molecule_chunks): pbar.add(len(tensors[0])) bbs_info.append(tensors) print('Merging batch features ... ', end='') else: bbs_info = pool.map(map_function, molecule_chunks) if self.verbose: print('[DONE]') # REDUCE: Concatenate the obtained tensors pool.close() pool.join() return self.concat_mol_features(bbs_info)
def model_fit(model, train, test, epochs=1, metrics=[], desc=None, debug=False): """Custom implementation of tf.keras.models.Model.fit. See https://github.com/tensorflow/tensorflow/issues/39448 for why this is necessary (how the f**k has this not been fixed?) Parameters ---------- model : tf.keras.models.Model Model to train. train : tf.data.Dataset Train dataset; should be batched and shuffled already if required. test : tf.data.Dataset Test dataset Keyword Args ------------ epochs : int Number of epochs to run. metrics : [callable(tf.Tensor, tf.Tensor) -> float] List of tensorflow metrics to evaluate. desc : str Description for display. debug : bool Whether to log debug information from optimizer.get_debug_summary(). """ strategy = tf.distribute.get_strategy() # Distribute datasets to replicas train = strategy.experimental_distribute_dataset(train) test = strategy.experimental_distribute_dataset(test) start_time = time.time() # Train def _train_step(batch): x, y = batch with tf.GradientTape() as tape: y_hat = model(x, training=True) loss = model.compiled_loss(y, y_hat) grads = tape.gradient(loss, model.trainable_variables) model.optimizer.apply_gradients(zip(grads, model.trainable_variables)) for m in metrics: m.update_state(y, y_hat) return loss @tf.function def train_step(batch): losses = strategy.run(_train_step, args=(batch, )) return strategy.reduce(tf.distribute.ReduceOp.SUM, losses, axis=None) # Test def _test_step(batch): x, y = batch y_hat = model(x, training=False) loss = model.compiled_loss(y, y_hat) for m in metrics: m.update_state(y, y_hat) return loss @tf.function def test_step(batch): losses = strategy.run(_test_step, args=(batch, )) return strategy.reduce(tf.distribute.ReduceOp.SUM, losses, axis=None) # Train/test loop def run_loop(dataset, step, callback=None): losses = [] times = [] for batch in dataset: losses.append(step(batch).numpy()) times.append(time.time() - start_time) if callback is not None: callback() metric_values = [m.result() for m in metrics] for m in metrics: m.reset_states() return losses, times, metric_values # List of stats to log stats = { "batch_loss": [], "batch_time": [], "loss": [], "val_loss": [], "epoch_time": [], } for m in metrics: stats[m.name] = [] stats["val_" + m.name] = [] # Debug if debug: trace = [] def log_debug(): trace.append( model.optimizer.get_debug_summary(model.trainable_variables)) else: log_debug = None # Epoch loop pbar = Progbar(epochs, unit_name='epoch') for _ in range(epochs): train_loss, train_time, train_metrics = run_loop(train, train_step, callback=log_debug) stats["batch_loss"] += train_loss stats["batch_time"] += train_time stats["loss"].append(np.mean(train_loss)) stats["epoch_time"].append(time.time() - start_time) for m, val in zip(metrics, train_metrics): stats[m.name].append(val) test_loss, test_time, test_metrics = run_loop(test, test_step) stats["val_loss"].append(np.mean(test_loss)) for m, val in zip(metrics, test_metrics): stats["val_" + m.name].append(val) pbar.add(1, values=[("train", stats["loss"][-1]), ("val", stats["val_loss"][-1])]) res = {k: np.array(v, dtype=np.float32) for k, v in stats.items()} if debug: res.update(model.optimizer.aggregate_debug_data(trace)) return res
def matrix(self, lastfm, tags=None, dim=3, save_to=None): ''' Computes a n-dimensional matrix where the (i_1, ... ,i_n)-th entry contains the number of tracks having all the i_1-th, ..., i_n-th tags (where the i's are the indexes in self.m_tags). Notes ----- To optimize performance, values are computed only with indexes in increasing order (which means, we only compute the number of tracks having tag-0 and tag-1, not vice-versa). This is something to keep in mind when indexing the matrix. To optimize memory, the matrix is saved in sparse format. DOK is the preferred sparse format for building and indexing, while COO is the preferred sparse format to perform mathematical operations). The dimension of the matrix captures the kind of queries which you will be able to perform. A matrix of dim=2 on tracks=['rock', 'pop', 'hip-hop'] will capture how many tracks have tags rock and pop, or pop and hip-hop, but not rock, pop and hip-hop at the same time. A matrix of dim=len(tags) will fully describe the database (or the subset of the database having the given tags). A matrix of dim>len(tags) will be rather pointless (but we won't prevent you from doing it). Parameters ---------- lastfm: LastFm, LastFm2Pandas Instance of tags database. Using LastFm2Pandas is strongly recommended here. tags: list List of tags to use. If None, all the tags will be used. dim: int The dimension of the matrix. save_to: str Filename or full path of the .npz file to save matrix and matrix tags. Use to load_from in the future. ''' # initialize matrix tags if tags is None: tags = lastfm.get_tags() else: tags = [tag for tag in tags if tag in lastfm.get_tags() ] # possibly purge inexistent tags # initialize matrix matrix = sparse.DOK( (len(tags), ) * dim, dtype=np.int32 ) # sparse dict-of-keys matrix (for easy creation, awful for calculations) # compute total number of steps to comatplotlibetion (see http://www.iosrjournals.org/iosr-jm/papers/Vol8-issue3/A0830110.pdf) n_steps = crazysum(n=len(tags), s=3, k=dim - 1) # check whether a progress bar is needed verbose = n_steps > 100 if verbose: progbar = Progbar(n_steps) # instantiate progress bar def count_intersect_tags(tags): tids_list = [lastfm.with_tag(tag) for tag in tags] tids_list.sort(key=len, reverse=True) tids = set( tids_list.pop() ) # start with shortest list of tids to improve performance; convert to set to be able to intersect for _ in range(len(tids_list)): tids = tids.intersection(tids_list.pop( )) # intersections performed from shortest list to longest return len(tids) # how many tids have all tags def count_intersect_tags_recursive( tags_idxs, dim ): # recursively iterate count_intersect_tags dim times; avoid repetitions such as 'rock AND pop AND folk' vs. 'rock AND folk AND pop' vs. 'folk AND pop AND rock' if dim >= 1: for i in range(tags_idxs[-1] + 1): count_intersect_tags_recursive(tags_idxs + (i, ), dim - 1) else: matrix[tags_idxs] = count_intersect_tags( np.take(tags, tags_idxs)) # add count to sparse matrix if verbose: progbar.add(1) # instantiate recursive loop for i in range(len(tags)): count_intersect_tags_recursive((i, ), dim - 1) matrix = matrix.to_coo() # convert to coordinate matrix if save_to is not None: # save matrix sparse.save_npz(save_to, matrix.to_coo( )) # default to compressed format (i.e. sparse format) # save matrix tags in serialized format with open(os.path.splitext(save_to)[0] + '.nfo', 'wb') as f: pickle.dump(tags, f) return matrix, tags
def fit_generator(self, generator=None, batch_size=None, epochs=1, verbose=True, logdir=None, ckptdir=None, n_recent=100, initial_epoch=0, steps_per_epoch=None, generator_val=None, batch_size_val=None, freq_val=10, **kwargs): # Initialize training if initial_epoch == 0: self.sess.run(self.init) if steps_per_epoch is None: steps_per_epoch = 1 # Progress freq_prog = 100 progbar = Progbar( steps_per_epoch, stateful_metrics=['loss', 'val loss', 'accuracy', 'val accuracy']) # Summary (Tensorboard) freq_log = 100 if not logdir is None: logdir = os.path.join( logdir, datetime.strftime(datetime.utcnow(), "%Y%m%d_%H%M%S")) flag_log = True train_writer = tf.summary.FileWriter(logdir, self.graph, flush_secs=5) else: flag_log = False # Checkpoint freq_ckpt = freq_val if not ckptdir is None: flag_ckpt = True fn_ckpt = os.path.join(ckptdir, 'WE_Label') # Save best model else: flag_ckpt = False def check_point(step): self.saver.save(self.sess, fn_ckpt, global_step=step) def load_best(): self.saver.restore(self.sess, tf.train.latest_checkpoint(ckptdir)) # History history = { 'train': deque(maxlen=n_recent), 'val': deque(maxlen=n_recent) } def is_plateau(): h = history['val'] if history['val'] else history['train'] return len(h) == h.maxlen and h[0] == max(h) best = deque(maxlen=1) def check_best(loss): if not best or loss <= best[0]: best.append(loss) return True else: return False # Validation if not generator_val is None: flag_val = True if batch_size_val is None: batch_size_val = batch_size else: flag_val = False # Stopping min_lr = 1E-5 # We must initialize all variables before we use them. self.sess.run(self.init) print('Initialized') for step in range(1, steps_per_epoch + 1): def should(freq): return step == 1 or step % freq == 0 batch_x, batch_y = self._preprocess_batch(*generator(batch_size)) feed_dict = {self.x: batch_x, self.y: batch_y} # We perform one update step by evaluating the optimizer op (including it # in the list of returned values for session.run() # Also, evaluate the merged op to get all summaries from the returned # "summary" variable. Feed metadata variable to session for visualizing # the graph in TensorBoard. _, loss, accuracy = self.sess.run( [self.optimizer, self.loss, self.accuracy], feed_dict=feed_dict) if flag_val and should(freq_val): batch_val_x, batch_val_y = self._preprocess_batch( *generator_val(batch_size_val)) feed_dict = {self.x: batch_val_x, self.y: batch_val_y} loss_val, accuracy_val = self.sess.run( [self.loss, self.accuracy], feed_dict=feed_dict) history['val'].append(-loss_val) else: history['train'].append(-loss) if verbose and should(freq_prog): progbar.update(step, [ ('loss', np.mean([loss])), ('val loss', np.mean([loss_val])), ('accuracy', np.mean([accuracy])), ('val accuracy', np.mean([accuracy_val])), ]) if flag_log and should(freq_log): for x in (self.sess.run(self.loss_summary, feed_dict={self.loss: loss}), self.sess.run(self.accuracy_summary, feed_dict={self.accuracy: accuracy})): train_writer.add_summary(x, step) if flag_val: for x in (self.sess.run(self.loss_val_summary, feed_dict={self.loss: loss_val}), self.sess.run( self.accuracy_val_summary, feed_dict={self.accuracy: accuracy_val})): train_writer.add_summary(x, step) if flag_ckpt and should(freq_ckpt): val_check = loss_val if flag_val else loss if check_best(val_check): print('\rSave BEST model at step {0} ({1:.3f})'.format( step, val_check), end='', flush=True) check_point(step) if is_plateau(): print('\nPlateau reached.'.format(step)) # Load best model load_best() # Try reduce learning rate self.learning_rate = self.learning_rate * 0.5 print('\nReduce lr to {}'.format( self.sess.run(self.learning_rate))) # Flush history [x.clear() for x in history.values()] if self.sess.run(self.learning_rate) < min_lr: print('\nLearning rate {0} reaches limit: < {1}.'.format( self.sess.run(self.learning_rate), min_lr)) break # Make sure to load the best model load_best() # Get fit embedding self.V, self.Uw, self.Ub = self.sess.run( [self.embeddings, self.weights, self.biases], feed_dict={}) # Combine weight and bias (intercept) for U (label embedding) self.U = np.concatenate((self.Uw, self.Ub[..., np.newaxis]), axis=1)
def val(sess, val_vars): sess.run([val_vars.it.initializer, tf.local_variables_initializer()]) val_progbar = Progbar(None, stateful_metrics=['loss', 'acc', 'num_word',]) preds = [] preds_b = [] preds_m = [] preds_s = [] preds_d = [] gts = [] indices = tf.argmax(val_vars.logits, 1) feed_dict = [val_vars.loss_metric, val_vars.acc_metric, val_vars.num_word_metric, indices, val_vars.labels] if FLAGS.bmsd: unshift = 1 if FLAGS.zero_based else 0 indices_b = tf.argmax(val_vars.logits_b, 1)+unshift indices_m = tf.argmax(val_vars.logits_m, 1)+unshift indices_s = tf.argmax(val_vars.logits_s, 1)+unshift indices_d = tf.argmax(val_vars.logits_d, 1)+unshift feed_dict.append(indices_b) feed_dict.append(indices_m) feed_dict.append(indices_s) feed_dict.append(indices_d) try: step = 0 while True: if FLAGS.bmsd: (loss, _), (acc, _), (num_word, _), pred, gt, pred_b, pred_m, pred_s, pred_d = sess.run(feed_dict) preds_b.append(pred_b) preds_m.append(pred_m) preds_s.append(pred_s) preds_d.append(pred_d) else: (loss, _), (acc, _), (num_word, _), pred, gt = sess.run(feed_dict) val_progbar.update(step, [('loss', loss), ('acc', acc), ('num_word', num_word), ]) preds.append(pred) gts.append(gt) except tf.errors.OutOfRangeError: logger.info('[*] Validation loss: %.4f, acc: %.4f, num_word: %.2f' % (loss, acc, num_word)) if FLAGS.bmsd: flat_preds = list(itertools.chain(*preds)) flat_gts = list(itertools.chain(*gts)) flat_preds_b = list(itertools.chain(*preds_b)) flat_preds_m = list(itertools.chain(*preds_m)) flat_preds_s = list(itertools.chain(*preds_s)) flat_preds_d = list(itertools.chain(*preds_d)) score, base_scores = evaluate(flat_gts, flat_preds) print('\n[!] Label Score: %.4f' % (score)) bmsd_score, bmsd_scores = evaluate_bmsd(flat_gts, flat_preds_b, flat_preds_m, flat_preds_s, flat_preds_d) print('BMSD Score: %.4f' % (bmsd_score)) final_score = 0.0 for r, s1, s2 in zip([1.0,1.2,1.3,1.4], base_scores, bmsd_scores): final_score += r * max(s1, s2) final_score /= 4.0 print('[!] Final Score: %.4f' % (final_score)) lfirst_score, lfirst_scores = evaluate_lfirst(flat_gts, flat_preds, flat_preds_b, flat_preds_m, flat_preds_s, flat_preds_d) print('[!] Label First Score: %.4f' % (lfirst_score)) else: flat_preds = list(itertools.chain(*preds)) flat_gts = list(itertools.chain(*gts)) score, base_scores = evaluate(flat_gts, flat_preds) print('\n[!] Label Score: %.4f' % (score)) return score
def main(): hvd.init() n_epochs = 10 batch_size = 5 step = len(im) // batch_size params = parse_args(PARSER.parse_args()) optimizer = tf.keras.optimizers.Adam(learning_rate=params.learning_rate) ce_loss = tf.keras.metrics.Mean(name='ce_loss') f1_loss = tf.keras.metrics.Mean(name='dice_loss') checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model) pb_i = Progbar(step, stateful_metrics=metrics_names) count = 0 for epoch in range(n_epochs): if count >= step: count = 0 features = im[epoch * batch_size:(epoch * batch_size) + batch_size] features = np.reshape(features, (len(features), features[0].shape[1], features[0].shape[2], features[0].shape[0])) features = features.astype('float32') labels = lb[epoch * batch_size:(epoch * batch_size) + batch_size] labels = np.reshape( labels, (len(labels), labels[0].shape[0], labels[0].shape[1], 1)) labels = labels.astype('float32') print(features.shape, labels.shape) print('Epoch {} out of epochs {}'.format(epoch, n_epochs)) for i, (features_, labels_) in enumerate(zip(features, labels)): with tf.GradientTape() as tape: output_map = model(features) crossentropy_loss, dice_loss = partial_losses( output_map, labels) added_losses = tf.add(crossentropy_loss, dice_loss, name='total_loss_ref') values = [('Xent', crossentropy_loss), ('added_losses', added_losses)] pb_i.add(1, values=values) # calculate the gradients using our tape and then update the # model weights tape = hvd.DistributedGradientTape(tape) gradients = tape.gradient(added_losses, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) # Calculate something wrong here # val_total_loss = 0 # val_total_acc = 0 # total_val_num = 0 # for bIdx, (val_X, val_y) in enumerate(val_batch): # if bIdx >= features.shape[0]: # break # y_pred = model(val_X, training=False) print('Xen: ', crossentropy_loss, dice_loss, added_losses)
# Initial Data class Data = Data_Generator(C) print('Start training') neg_flag = 0 # The start of ################################ Remove the data from last iteration for big_epoch_num in range(num_epochs): ########################################################################### #First CNN ########################################################################### first_loss = np.zeros(num_first_epochs) final_epoch_num = 0 for epoch_num in range(num_first_epochs): progbar = Progbar(Fov_num) print('Epoch of the First CNN {}/{}'.format(epoch_num + 1, num_first_epochs)) print('There are %d FOVs in total' % (Fov_num)) loss = 0 count = 0 change_matrix = 0 first_data_generator = itertools.cycle( Data.data_generator(FOV_Path, LABEL_Path, IMAGE_Path, epoch_num, big_epoch_num)) while count <= Fov_num: try: ## 设计思想,读取一次 训练一个batch,使用itertools, 一旦停止即进行下一个 for循环 # print('The %d epoch of the first CNN in %d big epoch'%(count, big_epoch_num)) Fov, Label, Image, x, y, z = next(first_data_generator)
def main(): # ------------------------------------------------------------------------------------ # Defining random seeds random_seed = 42 tf.random.set_seed(random_seed) np.random.seed(random_seed) rn.seed(random_seed) # ------------------------------------------------------------------------------------ # Loading data (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() height = 28 width = 28 n_channel = 1 x_train = x_train.astype('float32') / 255. x_test = x_test.astype('float32') / 255. x_train = x_train.reshape(x_train.shape[0], height, width, n_channel) x_test = x_test.reshape(x_test.shape[0], height, width, n_channel) # ------------------------------------------------------------------------------------ # Quantise the input data in q levels q_levels = 2 x_train_quantised = quantise(x_train, q_levels) x_test_quantised = quantise(x_test, q_levels) # ------------------------------------------------------------------------------------ # Creating input stream using tf.data API batch_size = 128 train_buf = 60000 train_dataset = tf.data.Dataset.from_tensor_slices( (x_train_quantised / (q_levels - 1), x_train_quantised.astype('int32'))) train_dataset = train_dataset.shuffle(buffer_size=train_buf) train_dataset = train_dataset.batch(batch_size) test_dataset = tf.data.Dataset.from_tensor_slices((x_test_quantised / (q_levels - 1), x_test_quantised.astype('int32'))) test_dataset = test_dataset.batch(batch_size) # ------------------------------------------------------------------------------------ # Create PixelCNN model inputs = keras.layers.Input(shape=(height, width, n_channel)) x = MaskedConv2D(mask_type='A', filters=128, kernel_size=7, strides=1)(inputs) for i in range(15): x = ResidualBlock(h=64)(x) x = keras.layers.Activation(activation='relu')(x) x = keras.layers.Conv2D(filters=128, kernel_size=1, strides=1)(x) x = keras.layers.Activation(activation='relu')(x) x = keras.layers.Conv2D(filters=q_levels, kernel_size=1, strides=1)(x) pixelcnn = keras.Model(inputs=inputs, outputs=x) # ------------------------------------------------------------------------------------ # Prepare optimizer and loss function lr_decay = 0.999995 learning_rate = 1e-3 optimizer = keras.optimizers.Adam(lr=learning_rate) compute_loss = keras.losses.CategoricalCrossentropy(from_logits=True) # ------------------------------------------------------------------------------------ @tf.function def train_step(batch_x, batch_y): with tf.GradientTape() as ae_tape: logits = pixelcnn(batch_x, training=True) loss = compute_loss(tf.squeeze(tf.one_hot(batch_y, q_levels)), logits) gradients = ae_tape.gradient(loss, pixelcnn.trainable_variables) gradients, _ = tf.clip_by_global_norm(gradients, 1.0) optimizer.apply_gradients(zip(gradients, pixelcnn.trainable_variables)) return loss # ------------------------------------------------------------------------------------ # Training loop n_epochs = 100 n_iter = int(np.ceil(x_train_quantised.shape[0] / batch_size)) for epoch in range(n_epochs): progbar = Progbar(n_iter) print('Epoch {:}/{:}'.format(epoch + 1, n_epochs)) for i_iter, (batch_x, batch_y) in enumerate(train_dataset): optimizer.lr = optimizer.lr * lr_decay loss = train_step(batch_x, batch_y) progbar.add(1, values=[('loss', loss)]) # ------------------------------------------------------------------------------------ # Test set performance test_loss = [] for batch_x, batch_y in test_dataset: logits = pixelcnn(batch_x, training=False) # Calculate cross-entropy (= negative log-likelihood) loss = compute_loss(tf.squeeze(tf.one_hot(batch_y, q_levels)), logits) test_loss.append(loss) print('nll : {:} nats'.format(np.array(test_loss).mean())) print('bits/dim : {:}'.format(np.array(test_loss).mean() / np.log(2))) # ------------------------------------------------------------------------------------ # Generating new images samples = np.zeros((100, height, width, n_channel), dtype='float32') for i in range(height): for j in range(width): logits = pixelcnn(samples) next_sample = tf.random.categorical(logits[:, i, j, :], 1) samples[:, i, j, 0] = (next_sample.numpy() / (q_levels - 1))[:, 0] fig = plt.figure(figsize=(10, 10)) for i in range(100): ax = fig.add_subplot(10, 10, i + 1) ax.matshow(samples[i, :, :, 0], cmap=matplotlib.cm.binary) plt.xticks(np.array([])) plt.yticks(np.array([])) plt.show() # ------------------------------------------------------------------------------------ # Filling occluded images occlude_start_row = 14 num_generated_images = 10 samples = np.copy(x_test_quantised[0:num_generated_images, :, :, :]) samples = samples / (q_levels - 1) samples[:, occlude_start_row:, :, :] = 0 fig = plt.figure(figsize=(10, 10)) for i in range(10): ax = fig.add_subplot(1, 10, i + 1) ax.matshow(samples[i, :, :, 0], cmap=matplotlib.cm.binary) plt.xticks(np.array([])) plt.yticks(np.array([])) for i in range(occlude_start_row, height): for j in range(width): logits = pixelcnn(samples) next_sample = tf.random.categorical(logits[:, i, j, :], 1) samples[:, i, j, 0] = (next_sample.numpy() / (q_levels - 1))[:, 0] fig = plt.figure(figsize=(10, 10)) for i in range(10): ax = fig.add_subplot(1, 10, i + 1) ax.matshow(samples[i, :, :, 0], cmap=matplotlib.cm.binary) plt.xticks(np.array([])) plt.yticks(np.array([])) plt.show()
def represent(self, molecules): """ provides coulomb matrix representation for input molecules. Parameters ---------- molecules: chemml.chem.Molecule object or array If list, it must be a list of chemml.chem.Molecule objects, otherwise we raise a ValueError. In addition, all the molecule objects must provide the XYZ information. Please make sure the XYZ geometry has been stored or optimized in advance. Returns ------- Pandas DataFrame A data frame with same number of rows as number of molecules will be returned. The exact shape of the dataframe depends on the type of CM as follows: - shape of Unsorted_Matrix (UM): (n_molecules, max_n_atoms**2) - shape of Unsorted_Triangular (UT): (n_molecules, max_n_atoms*(max_n_atoms+1)/2) - shape of eigenspectrums (E): (n_molecules, max_n_atoms) - shape of Sorted_Coulomb (SC): (n_molecules, max_n_atoms*(max_n_atoms+1)/2) - shape of Random_Coulomb (RC): (n_molecules, nPerm * max_n_atoms * (max_n_atoms+1)/2) """ # check input molecules if isinstance(molecules, (list, np.ndarray)): molecules = np.array(molecules) elif isinstance(molecules, Molecule): molecules = np.array([molecules]) else: msg = "The molecule must be a chemml.chem.Molecule object or a list of objects." raise ValueError(msg) if molecules.ndim > 1: msg = "The molecule must be a chemml.chem.Molecule object or a list of objects." raise ValueError(msg) self.n_molecules_ = molecules.shape[0] # max number of atoms based on the list of molecules if self.max_n_atoms_ == 'auto': try: self.max_n_atoms_ = max( [m.xyz.atomic_numbers.shape[0] for m in molecules]) except: msg = "The xyz representation of molecules is not available." raise ValueError(msg) # pool of processes if self.n_jobs == -1: self.n_jobs = cpu_count() pool = Pool(processes=self.n_jobs) # Create an iterator # http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks def chunks(l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] # find size of each batch batch_size = int(len(molecules) / self.n_jobs) if batch_size == 0: batch_size = 1 molecule_chunks = chunks(molecules, batch_size) # MAP: CM in parallel map_function = partial(self._represent) if self.verbose: print('featurizing molecules in batches of %i ...' % batch_size) pbar = Progbar(len(molecules), width=50) tensor_list = [] for tensors in pool.imap(map_function, molecule_chunks): pbar.add(len(tensors[0])) tensor_list.append(tensors) print('Merging batch features ... ', end='') else: tensor_list = pool.map(map_function, molecule_chunks) if self.verbose: print('[DONE]') # REDUCE: Concatenate the obtained tensors pool.close() pool.join() return pd.concat(tensor_list, axis=0, ignore_index=True)
def tensorise_molecules(molecules, max_degree=5, max_atoms=None, n_jobs=-1, batch_size=3000, verbose=True): """ Takes a list of molecules and provides tensor representation of atom and bond features. This representation is based on the "convolutional networks on graphs for learning molecular fingerprints" by David Duvenaud et al., NIPS 2015. Parameters ---------- molecules: chemml.chem.Molecule object or array If list, it must be a list of chemml.chem.Molecule objects, otherwise we raise a ValueError. In addition, all the molecule objects must provide the SMILES representation. We try to create the SMILES representation if it's not available. max_degree: int, optional (default=5) The maximum number of neighbour per atom that each molecule can have (to which all molecules will be padded), use 'None' for auto max_atoms: int, optional (default=None) The maximum number of atoms per molecule (to which all molecules will be padded), use 'None' for auto n_jobs: int, optional(default=-1) The number of parallel processes. If -1, uses all the available processes. batch_size: int, optional(default=3000) The number of molecules per process, bigger chunksize is preffered as each process will preallocate np.arrays verbose: bool, optional(default=True) The verbosity of messages. Notes ----- It is not recommended to set max_degree to `None`/auto when using `NeuralGraph` layers. Max_degree determines the number of trainable parameters and is essentially a hyperparameter. While models can be rebuilt using different `max_atoms`, they cannot be rebuild for different values of `max_degree`, as the architecture will be different. For organic molecules `max_degree=5` is a good value (Duvenaud et. al, 2015) Returns ------- atoms: array An atom feature array of shape (molecules, max_atoms, atom_features) bonds: array A bonds array of shape (molecules, max_atoms, max_degree) edges: array A connectivity array of shape (molecules, max_atoms, max_degree, bond_features) """ # TODO: # - fix python keyboardinterrupt bug: # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt # - replace progbar with proper logging # molecules if isinstance(molecules, list) or isinstance(molecules, np.ndarray): molecules = np.array(molecules) elif isinstance(molecules, Molecule): molecules = np.array([molecules]) else: msg = "The input molecules must be a chemml.chem.Molecule object or a list of objects." raise ValueError(msg) # pool of processes if n_jobs == -1: n_jobs = cpu_count() pool = Pool(processes=n_jobs) # Create an iterator #http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks def chunks(l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] molecule_chunks = chunks(molecules, batch_size) # MAP: Tensorise in parallel map_function = partial(tensorise_molecules_singlecore, max_degree=max_degree, max_atoms=max_atoms) if verbose: print('Tensorising molecules in batches of %i ...' % batch_size) pbar = Progbar(len(molecules), width=50) tensor_list = [] for tensors in pool.imap(map_function, molecule_chunks): pbar.add(tensors[0].shape[0]) tensor_list.append(tensors) print('Merging batch tensors ... ', end='') else: tensor_list = pool.map(map_function, molecule_chunks) if verbose: print('[DONE]') # REDUCE: Concatenate the obtained tensors pool.close() pool.join() return concat_mol_tensors(tensor_list, match_degree=max_degree != None, match_max_atoms=max_atoms != None)
def start_training(config): use_horovod = horovod_util.is_enabled() print("use_horovod:", use_horovod) if use_horovod: hvd = horovod_util.setup() rank = hvd.rank() else: rank = 0 ModelClass = config.NETWORK_CLASS network_kwargs = {key.lower(): val for key, val in config.NETWORK.items()} if "train_validation_saving_size".upper() in config.DATASET.keys(): use_train_validation_saving = config.DATASET.TRAIN_VALIDATION_SAVING_SIZE > 0 else: use_train_validation_saving = False if use_train_validation_saving: top_train_validation_saving_set_accuracy = 0 train_dataset = setup_dataset(config, "train", rank) print("train dataset num:", train_dataset.num_per_epoch) if use_train_validation_saving: train_validation_saving_dataset = setup_dataset(config, "train_validation_saving", rank) print("train_validation_saving dataset num:", train_validation_saving_dataset.num_per_epoch) validation_dataset = setup_dataset(config, "validation", rank) print("validation dataset num:", validation_dataset.num_per_epoch) graph = tf.Graph() with graph.as_default(): if ModelClass.__module__.startswith("lmnet.networks.object_detection"): model = ModelClass( classes=train_dataset.classes, num_max_boxes=train_dataset.num_max_boxes, is_debug=config.IS_DEBUG, **network_kwargs, ) else: model = ModelClass( classes=train_dataset.classes, is_debug=config.IS_DEBUG, **network_kwargs, ) global_step = tf.Variable(0, name="global_step", trainable=False) is_training_placeholder = tf.placeholder(tf.bool, name="is_training_placeholder") images_placeholder, labels_placeholder = model.placeholderes() output = model.inference(images_placeholder, is_training_placeholder) if ModelClass.__module__.startswith("lmnet.networks.object_detection"): loss = model.loss(output, labels_placeholder, global_step) else: loss = model.loss(output, labels_placeholder) opt = model.optimizer(global_step) if use_horovod: # add Horovod Distributed Optimizer opt = hvd.DistributedOptimizer(opt) train_op = model.train(loss, opt, global_step) metrics_ops_dict, metrics_update_op = model.metrics(output, labels_placeholder) # TODO(wakisaka): Deal with many networks. model.summary(output, labels_placeholder) summary_op = tf.summary.merge_all() metrics_summary_op, metrics_placeholders = executor.prepare_metrics(metrics_ops_dict) init_op = tf.global_variables_initializer() reset_metrics_op = tf.local_variables_initializer() if use_horovod: # add Horovod broadcasting variables from rank 0 to all bcast_global_variables_op = hvd.broadcast_global_variables(0) if use_train_validation_saving: saver = tf.train.Saver(max_to_keep=1) else: saver = tf.train.Saver(max_to_keep=config.KEEP_CHECKPOINT_MAX) if config.IS_PRETRAIN: all_vars = tf.global_variables() pretrain_var_list = [ var for var in all_vars if var.name.startswith(tuple(config.PRETRAIN_VARS)) ] print("pretrain_vars", [ var.name for var in pretrain_var_list ]) pretrain_saver = tf.train.Saver(pretrain_var_list, name="pretrain_saver") if use_horovod: # For distributed training session_config = tf.ConfigProto( gpu_options=tf.GPUOptions( allow_growth=True, visible_device_list=str(hvd.local_rank()) ) ) else: # TODO(wakisaka): For debug. # session_config = tf.ConfigProto( # gpu_options=tf.GPUOptions( # allow_growth=True, # per_process_gpu_memory_fraction=0.1 # ) # ) session_config = tf.ConfigProto() # tf.ConfigProto(log_device_placement=True) # TODO(wakisaka): XLA JIT # session_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 sess = tf.Session(graph=graph, config=session_config) sess.run([init_op, reset_metrics_op]) if rank == 0: train_writer = tf.summary.FileWriter(environment.TENSORBOARD_DIR + "/train", sess.graph) if use_train_validation_saving: train_val_saving_writer = tf.summary.FileWriter(environment.TENSORBOARD_DIR + "/train_validation_saving") val_writer = tf.summary.FileWriter(environment.TENSORBOARD_DIR + "/validation") if config.IS_PRETRAIN: print("------- Load pretrain data ----------") pretrain_saver.restore(sess, os.path.join(config.PRETRAIN_DIR, config.PRETRAIN_FILE)) sess.run(tf.assign(global_step, 0)) last_step = 0 # for recovery ckpt = tf.train.get_checkpoint_state(environment.CHECKPOINTS_DIR) if ckpt and ckpt.model_checkpoint_path: print("--------- Restore last checkpoint -------------") saver.restore(sess, ckpt.model_checkpoint_path) # saver.recover_last_checkpoints(ckpt.model_checkpoint_path) last_step = sess.run(global_step) # TODO(wakisaka): tensorflow v1.3 remain previous event log in tensorboard. # https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/python/training/supervisor.py#L1072 train_writer.add_session_log(SessionLog(status=SessionLog.START), global_step=last_step + 1) val_writer.add_session_log(SessionLog(status=SessionLog.START), global_step=last_step + 1) print("recovered. last step", last_step) if use_horovod: # broadcast variables from rank 0 to all other processes sess.run(bcast_global_variables_op) last_step = sess.run(global_step) # Calculate max steps. The priority of config.MAX_EPOCHS is higher than config.MAX_STEPS. if "MAX_EPOCHS" in config: max_steps = int(train_dataset.num_per_epoch / config.BATCH_SIZE * config.MAX_EPOCHS) else: max_steps = config.MAX_STEPS progbar = Progbar(max_steps) if rank == 0: progbar.update(last_step) for step in range(last_step, max_steps): images, labels = train_dataset.feed() feed_dict = { is_training_placeholder: True, images_placeholder: images, labels_placeholder: labels, } if step * ((step + 1) % config.SUMMARISE_STEPS) == 0 and rank == 0: # Runtime statistics for develop. # run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) # run_metadata = tf.RunMetadata() sess.run(reset_metrics_op) _, summary, _ = sess.run( [train_op, summary_op, metrics_update_op], feed_dict=feed_dict, # options=run_options, # run_metadata=run_metadata, ) # train_writer.add_run_metadata(run_metadata, "step: {}".format(step + 1)) train_writer.add_summary(summary, step + 1) metrics_values = sess.run(list(metrics_ops_dict.values())) metrics_feed_dict = {placeholder: value for placeholder, value in zip(metrics_placeholders, metrics_values)} metrics_summary, = sess.run( [metrics_summary_op], feed_dict=metrics_feed_dict, ) train_writer.add_summary(metrics_summary, step + 1) train_writer.flush() else: sess.run([train_op], feed_dict=feed_dict) to_be_saved = step == 0 or (step + 1) == max_steps or (step + 1) % config.SAVE_CHECKPOINT_STEPS == 0 if to_be_saved and rank == 0: if use_train_validation_saving: sess.run(reset_metrics_op) train_validation_saving_step_size = int(math.ceil(train_validation_saving_dataset.num_per_epoch / config.BATCH_SIZE)) print("train_validation_saving_step_size", train_validation_saving_step_size) current_train_validation_saving_set_accuracy = 0 for train_validation_saving_step in range(train_validation_saving_step_size): print("train_validation_saving_step", train_validation_saving_step) images, labels = train_validation_saving_dataset.feed() feed_dict = { is_training_placeholder: False, images_placeholder: images, labels_placeholder: labels, } if train_validation_saving_step % config.SUMMARISE_STEPS == 0: summary, _ = sess.run([summary_op, metrics_update_op], feed_dict=feed_dict) train_val_saving_writer.add_summary(summary, step + 1) train_val_saving_writer.flush() else: sess.run([metrics_update_op], feed_dict=feed_dict) metrics_values = sess.run(list(metrics_ops_dict.values())) metrics_feed_dict = { placeholder: value for placeholder, value in zip(metrics_placeholders, metrics_values) } metrics_summary, = sess.run( [metrics_summary_op], feed_dict=metrics_feed_dict, ) train_val_saving_writer.add_summary(metrics_summary, step + 1) train_val_saving_writer.flush() current_train_validation_saving_set_accuracy = sess.run(metrics_ops_dict["accuracy"]) if current_train_validation_saving_set_accuracy > top_train_validation_saving_set_accuracy: top_train_validation_saving_set_accuracy = current_train_validation_saving_set_accuracy print("New top train_validation_saving accuracy is: ", top_train_validation_saving_set_accuracy) _save_checkpoint(saver, sess, global_step, step) else: _save_checkpoint(saver, sess, global_step, step) if step == 0: # check create pb on only first step. minimal_graph = tf.graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(add_shapes=True), ["output"], ) pb_name = "minimal_graph_with_shape_{}.pb".format(step + 1) pbtxt_name = "minimal_graph_with_shape_{}.pbtxt".format(step + 1) tf.train.write_graph(minimal_graph, environment.CHECKPOINTS_DIR, pb_name, as_text=False) tf.train.write_graph(minimal_graph, environment.CHECKPOINTS_DIR, pbtxt_name, as_text=True) if step == 0 or (step + 1) % config.TEST_STEPS == 0: # init metrics values sess.run(reset_metrics_op) test_step_size = int(math.ceil(validation_dataset.num_per_epoch / config.BATCH_SIZE)) for test_step in range(test_step_size): images, labels = validation_dataset.feed() feed_dict = { is_training_placeholder: False, images_placeholder: images, labels_placeholder: labels, } if test_step % config.SUMMARISE_STEPS == 0: summary, _ = sess.run([summary_op, metrics_update_op], feed_dict=feed_dict) if rank == 0: val_writer.add_summary(summary, step + 1) val_writer.flush() else: sess.run([metrics_update_op], feed_dict=feed_dict) metrics_values = sess.run(list(metrics_ops_dict.values())) metrics_feed_dict = { placeholder: value for placeholder, value in zip(metrics_placeholders, metrics_values) } metrics_summary, = sess.run( [metrics_summary_op], feed_dict=metrics_feed_dict, ) if rank == 0: val_writer.add_summary(metrics_summary, step + 1) val_writer.flush() if rank == 0: progbar.update(step + 1) # training loop end. print("Done")
step = ckpt.step.numpy() val_metric = ckpt.metric.numpy() total_val_sample = 5000 progbar = None start_time = time.perf_counter() # Start training for image, cat_true, mask_true in train_dataset: ckpt.step.assign_add(1) step += 1 # On epoch start epoch_step = (step % TRAINING_PARAMETERS['steps_per_epoch']) + 1 if epoch_step == 1: print("Epoch {}/{}".format((step // TRAINING_PARAMETERS['steps_per_epoch']) + 1, TRAINING_PARAMETERS['num_epoch'])) progbar = Progbar(TRAINING_PARAMETERS['steps_per_epoch'], interval=1, stateful_metrics=['train_acc', 'train_meaniou']) total_loss, l_cate, l_mask = train_step(model, optimizer, loss_fn, image, cat_true, mask_true, train_acc, train_meaniou) values = [('train_loss', total_loss), ('train_cat_loss', l_cate), ('train_mask_loss', l_mask), ('train_acc', train_acc.result()), ('train_meaniou', train_meaniou.result())] progbar.update(epoch_step, values)
char = TimeDistributed(Flatten())(maxpool_out) char = Dropout(0.5)(char) output = concatenate([words, casing, char]) output = Bidirectional( LSTM(200, return_sequences=True, dropout=0.50, recurrent_dropout=0.25))(output) output = TimeDistributed(Dense(len(label2Idx), activation='softmax'))(output) model = Model(inputs=[words_input, casing_input, character_input], outputs=[output]) model.compile(loss='sparse_categorical_crossentropy', optimizer='nadam') model.summary() # plot_model(model, to_file='model.png') for epoch in range(epochs): print("Epoch %d/%d" % (epoch, epochs)) a = Progbar(len(train_batch_len)) for i, batch in enumerate(iterate_minibatches(train_batch, train_batch_len)): labels, tokens, casing, char = batch model.train_on_batch([tokens, casing, char], labels) a.update(i) a.update(i + 1) print(' ') model.save("models/model.h5") # Performance on dev dataset predLabels, correctLabels = tag_dataset(dev_batch) pre_dev, rec_dev, f1_dev = compute_f1(predLabels, correctLabels, idx2Label) print("Dev-Data: Prec: %.3f, Rec: %.3f, F1: %.3f" % (pre_dev, rec_dev, f1_dev))
d["business_id"] = "b_" + d["business_id"] d["review_id"] = "r_" + d["review_id"] review_data_raw[d["review_id"]] = d # Create NX graph G = nx.Graph() # Maintain a list of users, businesses and reviews in graphs # these are sets to make inclusion checking fast users_in_graph = set() business_in_graph = set() reviews_in_graph = set() # Create business nodes print("\nAdding businesses to graph") p = Progbar(len(business_data_raw)) for ii, b in enumerate(business_data_raw.values()): p.update(ii) if filter_state != "false" and b["state"].lower() != filter_state: continue G.add_node(b["business_id"], ntype="business") # Add business to set business_in_graph.add(b["business_id"]) # Create review links print("\nAdding reviews to graph") p = Progbar(len(review_data_raw)) for ii, r in enumerate(review_data_raw.values()): p.update(ii) if r["business_id"] not in business_in_graph:
#else: #best_loss2 = np.min(r_curr_loss2) start_time = time.time() class_mapping_inv = {v: k for k, v in class_mapping.items()} # Start training process print('Starting training') start_time = time.time() for epoch_num in range(num_epochs): progbar = Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) iter_num_aux = 0 iter_num = 0 i = True training = True validate = False #while(i): try: if(training): if len(rpn_accuracy_rpn_monitor) == len(train_imgs) and C.verbose: mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor)) / len(rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = []
def predict(self, x, batch_size=None, verbose=0, steps=1, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False): """ Model predicting on data yielded (generator). A predict() abstration function of TensorFlow 2 using the encoder and decoder models :param: See tensorflow.keras.Model.predict() :return: A numpy array(s) of predictions. References: Tal Weiss Deep Spelling Medium: https://machinelearnings.co/deep-spelling-9ffef96a24f6 Github: https://github.com/MajorTal/DeepSpell Vu Tran Sequence-to-Sequence Learning for Spelling Correction Github: https://github.com/vuptran/deep-spell-checkr """ try: enqueuer = GeneratorEnqueuer( x, use_multiprocessing=use_multiprocessing) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() steps_done = 0 if verbose == 1: print("Model Predict") progbar = Progbar(target=steps) predicts = [] while steps_done < steps: x = next(output_generator)[0] batch_size = len(x) # Encode the input as state vectors encoder_out, state_h, state_c = self.encoder.predict(x) dec_state = tf.concat([state_h, state_c], axis=-1) # Create batch of empty target sequences of length 1 character and populate # the first element of target sequence with the # start-of-sequence character target = np.zeros((batch_size, 1, self.tokenizer.vocab_size)) target[:, 0, self.tokenizer.SOS] = 1.0 # Sampling loop for a batch of sequences decoded_tokens = [''] * batch_size for _ in range(self.tokenizer.maxlen): # `char_probs` has shape (batch_size, 1, nb_target_chars) char_probs, dec_state = self.decoder.predict( [encoder_out, dec_state, target]) # Reset the target sequences. target = np.zeros( (batch_size, 1, self.tokenizer.vocab_size)) # Sample next character using argmax or multinomial mode sampled_chars = [] for i in range(batch_size): next_index = char_probs[i].argmax(axis=-1) next_char = self.tokenizer.decode([next_index]) decoded_tokens[i] += next_char sampled_chars.append(next_char) # Update target sequence with index of next character target[i, 0, next_index] = 1.0 stop_char = set(sampled_chars) if len(stop_char) == 1 and stop_char.pop( ) == self.tokenizer.EOS_TK: break # Sampling finished predicts.extend( [self.tokenizer.remove_tokens(x) for x in decoded_tokens]) steps_done += 1 if verbose == 1: progbar.update(steps_done) finally: enqueuer.stop() return predicts
def find_n_high(ds_name, encoder, paths, n, metric, negate=False): """The main function for executing network training. It loads the specified dataset iterator, saliency model, and helper classes. Training is then performed in a new session by iterating over all batches for a number of epochs. After validation on an independent set, the model is saved and the training history is updated. Args: ds_name (str): Denotes the dataset to be used during training. paths (dict, str): A dictionary with all path elements. """ w_filename_template = "/%s_%s_%s_weights.h5" # [encoder]_[ds_name]_[loss_fn_name]_weights.h5 (eval_ds, n_eval) = data.load_eval_dataset(ds_name, paths["data"]) print(">> Preparing model with encoder %s..." % encoder) model = MyModel(encoder, ds_name, "train") if "trained_weights" in paths: if os.path.exists(paths["trained_weights"]): weights_path = paths["trained_weights"] else: raise ValueError("could not find the specified weights file.\n specified weights: %s"%paths["trained_weights"]) else: weights_path = paths["weights"] + w_filename_template % (encoder, ds_name, loss_fn_name) if os.path.exists(weights_path): print("Weights are loaded!\n %s"%weights_path) else: download.download_pretrained_weights(paths["weights"], encoder, "salicon", loss_fn_name) model.load_weights(weights_path) del weights_path model.summary() # Preparing print("\n>> Start finding %d %s results for model on %s..." % (n, "worst" if negate else "best",ds_name.upper())) print(("Evaluation details:" + "\n{0:<4}Metric: {1}").format(" ", metric)) print("_" * 65) eval_progbar = Progbar(n_eval) min_heap = [] count = 0 sign = -1 if negate else 1 for eval_x, eval_fixs, eval_y_true, eval_ori_sizes, eval_filenames in eval_ds: eval_y_pred = test_step(eval_x, model) for pred, y_true, fixs, filename, ori_size in zip(eval_y_pred, eval_fixs, eval_y_true, eval_filenames.numpy(), eval_ori_sizes): pred = tf.expand_dims(data.postprocess_saliency_map(pred, ori_size), axis=0) fixs = tf.expand_dims(fixs, axis=0) y_true = tf.expand_dims(y_true, axis=0) score = _calc_metrics([metric], y_true, fixs, pred)[metric].numpy() * sign if count < n: count+=1 heapq.heappush(min_heap, (score, filename.decode("utf-8"))) else: heapq.heappushpop(min_heap, (score, filename.decode("utf-8"))) eval_progbar.add(eval_x.shape[0]) min_heap.sort(reverse=True) for s, n in min_heap: print(s, n)
def predict_generator(self, generator, steps, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0): """Generates predictions for the input samples from a data generator. The generator should return the same kind of data as accepted by `predict_on_batch`. generator = DataGenerator class that returns: x = Input data as a 3D Tensor (batch_size, max_input_len, dim_features) x_len = 1D array with the length of each data in batch_size # Arguments generator: Generator yielding batches of input samples or an instance of Sequence (tensorflow.keras.utils.Sequence) object in order to avoid duplicate data when using multiprocessing. steps: Total number of steps (batches of samples) to yield from `generator` before stopping. max_queue_size: Maximum size for the generator queue. workers: Maximum number of processes to spin up when using process based threading use_multiprocessing: If `True`, use process based threading. Note that because this implementation relies on multiprocessing, you should not pass non picklable arguments to the generator as they can't be passed easily to children processes. verbose: verbosity mode, 0 or 1. # Returns A numpy array(s) of predictions. # Raises ValueError: In case the generator yields data in an invalid format. """ self.model_pred._make_predict_function() is_sequence = isinstance(generator, Sequence) allab_outs = [] steps_done = 0 enqueuer = None try: if is_sequence: enqueuer = OrderedEnqueuer( generator, use_multiprocessing=use_multiprocessing) else: enqueuer = GeneratorEnqueuer( generator, use_multiprocessing=use_multiprocessing) enqueuer.start(workers=workers, max_queue_size=max_queue_size) output_generator = enqueuer.get() if verbose == 1: progbar = Progbar(target=steps) while steps_done < steps: x = next(output_generator) outs = self.predict_on_batch(x) if not isinstance(outs, list): outs = [outs] for i, out in enumerate(outs): encode = [ valab_out for valab_out in out if valab_out != -1 ] allab_outs.append([int(c) for c in encode]) steps_done += 1 if verbose == 1: progbar.update(steps_done) finally: if enqueuer is not None: enqueuer.stop() return allab_outs