def worker(fraction_used): num_samples_used = max(1, int(fraction_used * num_samples)) train_embs = datasets['train_dataset']['embs'][:num_samples_used] train_labels = get_targets_from_labels( datasets['train_dataset']['labels'][:num_samples_used], num_classes) return fit_model(train_embs, train_labels, val_embs, val_labels, global_step, num_classes, '%s_%s' % (datasets['name'], str(fraction_used)))
def evaluate_embeddings(self, algo, global_step, datasets): """Labeled evaluation.""" fractions = CONFIG.EVAL.CLASSIFICATION_FRACTIONS train_embs = datasets['train_dataset']['embs'] val_embs = datasets['val_dataset']['embs'] num_classes = DATASET_TO_NUM_CLASSES[datasets['name']] if not train_embs or not val_embs: logging.warn( 'All embeddings are NAN. Something is wrong with model.') return 1.0 val_labels = get_targets_from_labels(datasets['val_dataset']['labels'], num_classes) num_samples = len(datasets['train_dataset']['embs']) def worker(fraction_used): num_samples_used = max(1, int(fraction_used * num_samples)) train_embs = datasets['train_dataset']['embs'][:num_samples_used] train_labels = get_targets_from_labels( datasets['train_dataset']['labels'][:num_samples_used], num_classes) return fit_model(train_embs, train_labels, val_embs, val_labels, global_step, num_classes, '%s_%s' % (datasets['name'], str(fraction_used))) val_scores = [] with cf.ThreadPoolExecutor(max_workers=len(fractions)) as executor: results = executor.map(worker, fractions) for (fraction, (train_score, val_score)) in zip(fractions, results): prefix = '%s_%s' % (datasets['name'], str(fraction)) logging.info( '[Global step: {}] Event Completion {} Fraction Train ' 'Score: {:.3f},'.format(global_step.numpy(), prefix, train_score)) logging.info( '[Global step: {}] Event Completion {} Fraction Val ' 'Score: {:.3f},'.format(global_step.numpy(), prefix, val_score)) tf.summary.scalar('event_completion/train_%s_score' % prefix, train_score, step=global_step) tf.summary.scalar('event_completion/val_%s_score' % prefix, val_score, step=global_step) val_scores.append(val_score) return val_scores[-1]