def evaluate_single_thread(self, model, mode='test', max_nr_batches=None): metrics = self.init_metrics() nr_batches = 0 batch_size = 128 dataset = load_dataset(self.dataset, mode).batch(batch_size) if max_nr_batches is not None: dataset = dataset.take(max_nr_batches) for batch in dataset: x = batch['x'] user_ids = batch['user_id'] predictions = model.predict(x, user_ids) print('\rBatch nr {} predicted'.format(nr_batches + 1), end='\r') result = self.calc_recommendation_metrics(predictions, batch) nr_batches += 1 metrics = Evaluation.update_metrics(metrics, result) metrics = self.collect_metrics(metrics) metrics['name'] = model.get_name() for k, v in model.get_params().items(): metrics[k] = v return metrics
def main(): args = parse_args() seed = 6 fold_idx = 0 print('Loading data...') x_data, y_data = load_dataset(args.dir, duration=args.win_len, shift=args.shift, use_amp=args.use_amp) x_test, y_test = load_dataset(args.test, duration=args.win_len, shift=args.shift, use_amp=args.use_amp) print(f'xdata shape: {x_data.shape}') # Change to 2d, (N,C,W) to (N,C,H,W), H=1 old_shape = x_data.shape x_data = x_data.reshape(old_shape[0], 1, old_shape[1], old_shape[2]) old_shape = x_test.shape x_test = x_test.reshape(old_shape[0], 1, old_shape[1], old_shape[2]) print(f'xdata new shape: {x_data.shape}') if args.kfold <= 0: x_train, x_val, y_train, y_val = train_test_split(x_data, y_data, test_size=0.33, random_state=seed) acc = train(args, x_train, y_train, x_val, y_val, x_test, y_test, fold_idx) logging.info("accuracy score is:%s" % acc) else: skf = StratifiedKFold(n_splits=args.kfold, shuffle=args.shuffle, random_state=seed) cv_scores = [] for train_index, val_index in skf.split(x_data, y_data): print("train", train_index, "val", val_index) x_train, x_val = x_data[train_index], x_data[val_index] y_train, y_val = y_data[train_index], y_data[val_index] acc = train(args, x_train, y_train, x_val, y_val, x_test, y_test, fold_idx) cv_scores.append(acc) fold_idx += 1 logging.info("%.4f%% (+/- %.4f%%)" % (np.mean(cv_scores), np.std(cv_scores)))
def evaluate(self, model, mode='test', max_nr_batches=None): nr_batches = 0 dataset = load_dataset(self.dataset, mode).batch(128) if max_nr_batches is not None: dataset = dataset.take(max_nr_batches) metrics = dict() for batch in dataset: x = batch['x'] user_ids = batch['user_id'] predictions = model.predict(x, user_ids) print('Batch nr {} predicted'.format(nr_batches + 1)) self.input_q.put((batch, predictions, nr_batches)) nr_batches += 1 print('waiting for queue') self.input_q.join() self.output_q.put(None) print('processing results') i = 0 while True: result = self.output_q.get() if result is None: break i += 1 printProgressBar(i, nr_batches, 'Evaluating {}'.format(model.get_name()), length=60) for k in result: metrics.setdefault(k, 0) metrics[k] += result[k] self.output_q.task_done() metrics = {key: (v / nr_batches) for (key, v) in metrics.items()} metrics['name'] = model.get_name() for k, v in model.get_params().items(): metrics[k] = v return metrics
import tensorflow as tf from models.ConstraintAutoRec import ConstraintAutoRec import datetime import os from utils.common import movie_lens, load_dataset model = ConstraintAutoRec(movie_lens['dimensions']) model.train(load_dataset(movie_lens), movie_lens['train']['records']) # today = datetime.date.today() # directory = 'saved_models/' + str(today) + '/' # if not os.path.exists(directory): # os.makedirs(directory) # model.save(directory)
def main(cfg): #Configuramos para utitilizar toda la memoria de los dispoisitvo GPU gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: try: # Currently, memory growth needs to be the same across GPUs for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) except RuntimeError as e: # Memory growth must be set before GPUs have been initialized print(e) # TODO: Habilitar loggers # Definimos la ubicación donde se almacena los logs de Tensorboard train_summary_writer = tf.summary.create_file_writer(cfg.dirs.train_log) # Preparamos el DATASET # Cargamos los datos desde el fichero CSV pids, fids = load_dataset(cfg.dirs.csv_file, cfg.dirs.images) # Obtenemos todas las etiquetas unique_pids = np.unique(pids) print ("Etiquetas únicas: {}".format(len(unique_pids))) # Preparamos un dataset donde en cada época se se cubran todos los valores de PID # y si se distribuyan uniformemente. dataset = tf.data.Dataset.from_tensor_slices(unique_pids) if len(unique_pids) < cfg.model.batch.P: unique_pids = np.tile(unique_pids, int(np.ceil(cfg.model.batch.P / len(unique_pids)))) dataset = tf.data.Dataset.from_tensor_slices(unique_pids) # Cogemos valores aleatorios dataset = dataset.shuffle(len(unique_pids)) # Forzamos que el tamaño del dataset sea múltiplo de batch-size dataset = dataset.take((len(unique_pids) // cfg.model.batch.P) * cfg.model.batch.P) dataset = dataset.repeat(None) # Para cada PID obtenemos el cfg.model.batch.K imagenes dataset = dataset.map(lambda pid: sample_k_fids_for_pid( pid, all_fids=fids, all_pids=pids, batch_k=cfg.model.batch.K)) # Desagrupamos los cfg.model.batch.K para una mejor carga de las imágenes dataset = dataset.unbatch() # Ahora cargamos las imágenes transformandolas al tamaño del emb_modelo net_input_size = (cfg.model.input.height, cfg.model.input.width) pre_crop_size = (cfg.model.crop.height, cfg.model.crop.width) # Comprobamos si queremos hacer el crop image_size = pre_crop_size if cfg.model.crop else net_input_size # Redimensionamos las imágenes dataset = dataset.map(lambda fid, pid: fid_to_image ( fid, pid, cfg.dirs.images, image_size), num_parallel_calls=cfg.loading_threads) # Si se ha habilitado el CROP redimensionamos al tamaño de red if cfg.model.crop: dataset = dataset.map(lambda im, fid, pid: (tf.image.random_crop(im, net_input_size + (3,)), fid, pid)) #Carga de y un model Predefinido y preparación para el entrenamiento # Definimos la fase de nuestro entorno TF 0 = test, 1= train tf.keras.backend.set_learning_phase(1) emb_model = EmbeddingModel(cfg) # Agrupamos el dataset en los batch_size # y preprocesamos la imagen para prepararlo para el emb_modelo batch_size = cfg.model.batch.P * cfg.model.batch.K dataset = dataset.map(lambda im, fid, pid: (emb_model.preprocess_input(im), fid, pid)) dataset = dataset.batch(batch_size) print ('Batch-size: {}'.format(batch_size)) # Preparación de los siguientes batch # Esto mejora la latencia y el rendimiento en el coste computacional de usar # memoria adicional para almacenar los siguientes batch dataset = dataset.prefetch(2) # Establecemos el optimizador y la programación ratio de aprendizaje (learning-rate schedule) if 0 <=cfg.model.fit.decay_start_iteration <cfg.model.fit.epochs: cfg.model.fit.lr = tf.optimizers.schedules.PolynomialDecay(cfg.model.fit.lr,cfg.model.fit.epochs, end_learning_rate=1e-7) else: cfg.model.fit.lr =cfg.model.fit.lr if cfg.model.fit.optimizer== 'adam': optimizer= tf.keras.optimizers.Adam(cfg.model.fit.lr) elif cfg.model.fit.optimizer== 'SGD': optimizer= tf.keras.optimizers.SGD(cfg.model.fit.lr, momentum=0.9) else: raise NotImplementedError('Optimizador no válido {}'.format(cfg.model.fit.optimizer)) # Iniciamos el entrenamiento start_step = 0 dataset_iter = iter(dataset) # Definimos los checkpoint ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=emb_model) manager = tf.train.CheckpointManager(ckpt, cfg.dirs.checkpoint, max_to_keep=3) # Recuperamos el último checkpoint ckpt.restore(manager.latest_checkpoint) if manager.latest_checkpoint: print("Recuperado desde {}".format(manager.latest_checkpoint)) else: print("Inicializado desde inicio.") # Almacenamos los datos para tensorboard tf.summary.trace_on(graph=True, profiler=True) # Función para facilmente cambiar los estados del optimizador @contextlib.contextmanager def options(options): old_opts = tf.config.optimizer.get_experimental_options() tf.config.optimizer.set_experimental_options(options) try: yield finally: tf.config.optimizer.set_experimental_options(old_opts) @tf.function(experimental_relax_shapes=True) def train_step(images, pids, iteration ): cfg = emb_model.cfg with tf.GradientTape() as tape: # Obtenemos de cada batch los correspondientes vectores # de característias batch_embedding = emb_model(images) # Realizamos la norma de orden 2 si procede if emb_model.l2_embedding: batch_embedding = tf.nn.l2_normalize(batch_embedding, -1) else: batch_embedding = batch_embedding # Aplicacmos una función de perdia if cfg.model.fit.loss == 'semi_hard_triplet': embedding_loss = triplet_semihard_loss(batch_embedding, pids, cfg.model.fit.margin) elif cfg.model.fit.loss == 'hard_triplet': embedding_loss = batch_hard(batch_embedding, pids, cfg.model.fit.margin, cfg.model.fit.metric) elif cfg.model.fit.loss == 'lifted_loss': embedding_loss = lifted_loss(pids, batch_embedding, margin=cfg.model.fit.margin) elif cfg.model.fit.loss == 'contrastive_loss': assert batch_size % 2 == 0 assert cfg.model.batch.K == 4 ## Can work with other number but will need tuning contrastive_idx = np.tile([0, 1, 4, 3, 2, 5, 6, 7], cfg.model.batch.P // 2) for i in range(cfg.model.batch.P // 2): contrastive_idx[i * 8:i * 8 + 8] += i * 8 contrastive_idx = np.expand_dims(contrastive_idx, 1) batch_embedding_ordered = tf.gather_nd(batch_embedding, contrastive_idx) pids_ordered = tf.gather_nd(pids, contrastive_idx) # batch_embedding_ordered = tf.Print(batch_embedding_ordered,[pids_ordered],'pids_ordered :: ',summarize=1000) embeddings_anchor, embeddings_positive = tf.unstack( tf.reshape(batch_embedding_ordered, [-1, 2, cfg.model.embedding_dim]), 2, 1) # embeddings_anchor = tf.Print(embeddings_anchor,[pids_ordered,embeddings_anchor,embeddings_positive,batch_embedding,batch_embedding_ordered],"Tensors ", summarize=1000) fixed_labels = np.tile([1, 0, 0, 1], cfg.model.batch.P // 2) # fixed_labels = np.reshape(fixed_labels,(len(fixed_labels),1)) # print(fixed_labels) labels = tf.constant(fixed_labels) # labels = tf.Print(labels,[labels],'labels ',summarize=1000) embedding_loss = contrastive_loss(labels, embeddings_anchor, embeddings_positive, margin=cfg.model.fit.margin) elif cfg.model.fit.loss == 'angular_loss': embeddings_anchor, embeddings_positive = tf.unstack( tf.reshape(batch_embedding, [-1, 2, cfg.model.embedding_dim]), 2, 1) # pids = tf.Print(pids, [pids], 'pids:: ', summarize=100) pids, _ = tf.unstack(tf.reshape(pids, [-1, 2, 1]), 2, 1) # pids = tf.Print(pids,[pids],'pids:: ',summarize=100) # Añadimos el parámetro del ángulo máximo que puede # forma epn y ean embedding_loss = angular_loss(pids, embeddings_anchor, embeddings_positive, degree= cfg.model.fit.alpha, batch_size=cfg.model.batch.P, with_l2reg=True) elif cfg.model.fit.loss == 'npairs_loss': assert cfg.model.batch.K == 2 ## Single positive pair per class embeddings_anchor, embeddings_positive = tf.unstack( tf.reshape(batch_embedding, [-1, 2, cfg.model.embedding_dim]), 2, 1) pids, _ = tf.unstack(tf.reshape(pids, [-1, 2, 1]), 2, 1) pids = tf.reshape(pids, [-1]) embedding_loss = npairs_loss(pids, embeddings_anchor, embeddings_positive) else: raise NotImplementedError('Invalid Loss {}'.format(cfg.model.fit.loss)) loss_mean = tf.reduce_mean(embedding_loss) gradients = tape.gradient(loss_mean, emb_model.trainable_variables) optimizer.apply_gradients(zip(gradients, emb_model.trainable_variables)) # Almacenamos los datos para tensorboard with train_summary_writer.as_default(): tf.summary.scalar('Loss mean', loss_mean, step=iteration) # tf.summary.image("Training data", images, step=iteration) # tf.summary.scalar('Learning Rate', optimizer.lr, step=iteration) return embedding_loss #print('Starting training from iteration {}.'.format(start_step)) with lb.Uninterrupt(sigs=[SIGINT, SIGTERM], verbose=True) as u: for i in range(ckpt.step.numpy(),cfg.model.fit.epochs): # for batch_idx, batch in enumerate(): start_time = time.time() images, fids, pids = next(dataset_iter) # strategy = tf.distribute.MirroredStrategy() # strategy = tf.distribute.OneDeviceStrategy(device="/gpu:0") # with strategy.scope(): # TODO: Leer el documento https://www.tensorflow.org/guide/distributed_training#using_tfdistributestrategy_with_custom_training_loops batch_loss = train_step(images, pids, i) elapsed_time = time.time() - start_time seconds_todo = (cfg.model.fit.epochs - i) * elapsed_time print('iter:{:6d}, loss min|avg|max: {:.3f}|{:.3f}|{:6.3f}, ETA: {} ({:.2f}s/it)'.format( i, tf.reduce_min(batch_loss).numpy(),tf.reduce_mean(batch_loss).numpy(),tf.reduce_max(batch_loss).numpy(), # cfg.model.batch.K - 1, float(b_prec_at_k), timedelta(seconds=int(seconds_todo)), elapsed_time)) ckpt.step.assign_add(1) if (cfg.checkpoint_frequency> 0 and i % cfg.checkpoint_frequency== 0): #uncomment if you want to save the emb_model weight separately #emb_model.save_weights(os.path.join(cfg.dirs.checkpoint, 'emb_model_weights_{0:04d}.w'.format(i))) manager.save() # Stop the main-loop at the end of the step, if requested. if u.interrupted: log.info("Interrupted on request!") break
def prepare_training(train_dir, test_dir, train_cfg): fs = 26 # duration = 4.04 # int(4.04 * 26) = 105 # duration = 8.08 # int(8.08 * 26) = 210 duration = 8.35 # int(8.35 * 26) = 217 shift = 2 use_amp = True filter_outlier = True lp_filter = False seed = 17 cv_size = 0.33 train_x, train_y = load_dataset(train_dir, fs=fs, duration=duration, shift=shift, use_amp=use_amp, filter_outlier=filter_outlier, lp_filter=lp_filter) test_x, test_y = load_dataset(test_dir, fs=fs, duration=duration, shift=shift, use_amp=use_amp, filter_outlier=filter_outlier, lp_filter=lp_filter) logger.info(f'Train x shape: {train_x.shape}') logger.info(f'Test x shape: {test_x.shape}') # Normalize norm = None # train_x, norm = data_normalize(train_x) # test_x = normalize_data(test_x, norm) train_x = np.transpose(train_x, (0, 2, 1)) test_x = np.transpose(test_x, (0, 2, 1)) # train_y = train_y.reshape((-1, 1)) # test_y = test_y.reshape((-1, 1)) logger.info(f'Train x transposed shape: {train_x.shape}') logger.info(f'Train y shape: {train_y.shape}') logger.info(f'Test x transposed shape: {test_x.shape}') # Change to 2d, (N,C,W) to (N,C,H,W), H=1 old_shape = train_x.shape train_x = train_x.reshape(old_shape[0], old_shape[1], 1, old_shape[2]) old_shape = test_x.shape test_x = test_x.reshape(old_shape[0], old_shape[1], 1, old_shape[2]) # Training kfold = train_cfg['kfold'] if kfold > 1: skf = StratifiedKFold(n_splits=kfold, shuffle=False, random_state=seed) fold_idx = 0 accuracy_scores = [] for train_index, val_index in skf.split(train_x, train_y): print(f'KFold [{fold_idx + 1}/{kfold}]...') print("train", train_index, "val", val_index) x_train, x_cv = train_x[train_index], train_x[val_index] y_train, y_cv = train_y[train_index], train_y[val_index] model_path = CNN_MODEL_PATH.with_suffix(f'.cv.{fold_idx}.pth') accuracy = start_training(x_train, y_train, x_cv, y_cv, test_x, test_y, norm, model_path, train_cfg) accuracy_scores.append(accuracy) fold_idx += 1 for i, accuracy in enumerate(accuracy_scores, 1): print(f'K-Fold Accuracy for fold {i}: {accuracy:0.4f}') print(f'KFold Accuracy mean: {np.mean(accuracy_scores):0.2f} ' f'+/- {np.std(accuracy_scores):0.2f}') else: # Shuffle train_x, train_y = shuffle(train_x, train_y, random_state=seed) # Split cv set from train set train_x, cv_x, train_y, cv_y = train_test_split(train_x, train_y, test_size=cv_size, random_state=seed, stratify=train_y) model_path = CNN_MODEL_PATH start_training(train_x, train_y, cv_x, cv_y, test_x, test_y, norm, model_path, train_cfg)