def __init__(self, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._alg_name = kwargs.get('alg') # data args self._raw_dataset_dir = self._data_kwargs.get('raw_dataset_dir') self._test_size = self._data_kwargs.get('test_size') # logging self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._logger.info(kwargs) # Model's Args self._verified_percentage = self._model_kwargs.get( 'verified_percentage') self._seq_len = self._model_kwargs.get('seq_len') self._horizon = self._model_kwargs.get('horizon') self._nodes = self._model_kwargs.get('num_nodes') # Test's args self._run_times = self._test_kwargs.get('run_times') # Load data self._data = np.load(self._raw_dataset_dir)['data']
def __init__(self, obu_idx, total_obu_num, GLOBAL_ID_OBU_i_PK_i_map): super().__init__() self.ID_OBU_i = get_ID_obu_i(obu_idx) self.total_num = total_obu_num self.idx = str(obu_idx) self.logger = get_logger("OBU_" + self.idx) # ========= 生成属性 ========= self.r_i = random.randint(1, self.P) self.x_i, self.y_i, self.PK_i = get_x_i_y_i_PK_i( self.P, self.ID_OBU_i, self.s) GLOBAL_ID_OBU_i_PK_i_map[self.ID_OBU_i] = self.PK_i self.GLOBAL_ID_OBU_i_PK_i_map = GLOBAL_ID_OBU_i_PK_i_map self.R_i = self.r_i * self.P self.Ri_map = dict() self.R = 0 self.logger.debug("x_i: {}, y_i: {}, PK_i: {}, R_i: {}".format( self.x_i, self.y_i, self.PK_i, self.R_i)) # ========= 信号量定义 ======= self.sem_r_receive = Semaphore(0) self.sem_rsp_msg = Semaphore(0) # ========= 网络设置 ======== self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) # ========= 回包 ========= self.rsp_msg = None
def __init__(self, **kwargs): self._kwargs = kwargs self._alg = kwargs.get('alg') self._data_kwargs = kwargs.get('data') self._train_kwargs = kwargs.get('train') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._base_dir = kwargs.get('base_dir') self._epochs = self._train_kwargs.get('epochs') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._writer = tf.summary.FileWriter(self._log_dir) self._logger.info(kwargs) self._mon_ratio = float(self._kwargs.get('mon_ratio')) # Model's args self._seq_len = int(self._model_kwargs.get('seq_len')) self._horizon = int(self._model_kwargs.get('horizon')) self._input_dim = int(self._model_kwargs.get('input_dim')) self._nodes = int(self._model_kwargs.get('num_nodes')) self._rnn_units = int(self._model_kwargs.get('rnn_units')) self._drop_out = float(self._train_kwargs.get('dropout')) # Test's args self._flow_selection = self._test_kwargs.get('flow_selection') self._run_times = self._test_kwargs.get('run_times') # Data preparation self._day_size = self._data_kwargs.get('day_size')
def __init__(self, adj_mx, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._model_kwargs = kwargs.get('model') self._train_kwargs = kwargs.get('train') self.max_grad_norm = self._train_kwargs.get('max_grad_norm', 1.) # logging. self._log_dir = self._get_log_dir(kwargs) self._writer = SummaryWriter(self._log_dir + '/runs/') log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) # data set self._data = utils.load_dataset(**self._data_kwargs) self.standard_scaler = self._data['scaler'] self.num_nodes = int(self._model_kwargs.get('num_nodes', 1)) self.input_dim = int(self._model_kwargs.get('input_dim', 1)) self.seq_len = int(self._model_kwargs.get('seq_len')) # for the encoder self.output_dim = int(self._model_kwargs.get('output_dim', 1)) self.use_curriculum_learning = bool( self._model_kwargs.get('use_curriculum_learning', False)) self.horizon = int(self._model_kwargs.get('horizon', 1)) # for the decoder # setup model garnn_model = GARNNModel(adj_mx, self._logger, **self._model_kwargs) self.garnn_model = garnn_model.cuda() if torch.cuda.is_available() else garnn_model self._logger.info("Model created") self._epoch_num = self._train_kwargs.get('epoch', 0) if self._epoch_num > 0: self.load_model()
def __init__(self, GLOBAL_ID_OBU_i_PK_i_map, prime_nums): super().__init__() self.logger = get_logger("AMF") self.prime_nums = prime_nums # ========= 生成属性 ========= self.delta_i_list = list() self.R = 0 self.R_AMF = 0 self.X_AMF = 0 self.W = 0 self.ID_OBU_i_V_i_map = dict() self.GLOBAL_ID_OBU_i_PK_i_map = GLOBAL_ID_OBU_i_PK_i_map self.R_i_msg_map = dict() self.r_AMF = 0 self.GK = 0 self.x_amf, self.y_amf, self.PK_AMF = get_x_i_y_i_PK_i( self.P, self.ID_AMF, self.s) # ========= 信号量定义 ======= self.sem_qm_msg_receive = Semaphore(0) # ========= 网络设置 ======== self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
def create_logger(self): if self.logger or self.writer: return if not os.path.exists(self.log_root): os.makedirs(self.log_root) self.writer = SummaryWriter(log_dir=self.log_root) opt_str = option_to_string(self.opt) with open(os.path.join(self.log_root, 'config.txt'), 'w') as f: f.writelines(opt_str) self.logger = get_logger(self.log_root)
def train_toy(toy, load=True, nb_steps=20, nb_flow=1, folder=""): device = "cpu" logger = utils.get_logger(logpath=os.path.join(folder, toy, 'logs'), filepath=os.path.abspath(__file__)) logger.info("Creating model...") model = UMNNMAFFlow(nb_flow=nb_flow, nb_in=2, hidden_derivative=[50, 50, 50, 50], hidden_embedding=[50, 50, 50, 50], embedding_s=10, nb_steps=nb_steps, device=device).to(device) logger.info("Model created.") opt = torch.optim.Adam(model.parameters(), 1e-3, weight_decay=1e-5) if load: logger.info("Loading model...") model.load_state_dict(torch.load(folder + toy + '/model.pt')) model.train() opt.load_state_dict(torch.load(folder + toy + '/ADAM.pt')) logger.info("Model loaded.") nb_samp = 1000 batch_size = 100 x_test = torch.tensor(toy_data.inf_train_gen(toy, batch_size=1000)).to(device) x = torch.tensor(toy_data.inf_train_gen(toy, batch_size=1000)).to(device) for epoch in range(10000): ll_tot = 0 start = timer() for j in range(0, nb_samp, batch_size): cur_x = torch.tensor( toy_data.inf_train_gen(toy, batch_size=batch_size)).to(device) ll, z = model.compute_ll(cur_x) ll = -ll.mean() ll_tot += ll.detach() / (nb_samp / batch_size) loss = ll opt.zero_grad() loss.backward() opt.step() end = timer() ll_test, _ = model.compute_ll(x_test) ll_test = -ll_test.mean() logger.info( "epoch: {:d} - Train loss: {:4f} - Test loss: {:4f} - Elapsed time per epoch {:4f} (seconds)" .format(epoch, ll_tot.item(), ll_test.item(), end - start)) if (epoch % 100) == 0: summary_plots(x, x_test, folder, epoch, model, ll_tot, ll_test) torch.save(model.state_dict(), folder + toy + '/model.pt') torch.save(opt.state_dict(), folder + toy + '/ADAM.pt')
def __init__(self, catalog, override=False, schema='public'): self.conn = SQLConnection('xmm', schema) self.conn.autocommit = True self.working_format = 'votable' self.logger = get_logger('ICFx', log_filename='/tmp/icfx.log') self.logger.setLevel(logging.DEBUG) self.set_catalog(catalog) self.override = override self.source_catalogs = self.conn.execute_set(""" select distinct source_catalog from fields;""", quiet=False) self.source_catalogs = [x[0] for x in self.source_catalogs] sqllist.load_defaults() sqllist.GLOBALS['c'] = self.catalog.CATALOG
def __init__(self, data_type, LOAD_INITIAL, adj_mx, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._model_kwargs = kwargs.get('model') self._train_kwargs = kwargs.get('train') self.max_grad_norm = self._train_kwargs.get('max_grad_norm', 1.) # logging. self._log_dir = self._get_log_dir(kwargs) # self._writer = SummaryWriter('runs/' + self._log_dir) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) # data set self._data = utils.load_dataset(**self._data_kwargs) self.standard_scaler = self._data['scaler'] self.num_nodes = int(self._model_kwargs.get('num_nodes', 1)) self.input_dim = int(self._model_kwargs.get('input_dim', 1)) self.seq_len = int( self._model_kwargs.get('seq_len')) # for the encoder self.output_dim = int(self._model_kwargs.get('output_dim', 1)) self.use_curriculum_learning = bool( self._model_kwargs.get('use_curriculum_learning', False)) self.horizon = int(self._model_kwargs.get('horizon', 1)) # for the decoder # setup model # dcrnn_model = DCRNNModel(adj_mx, self._logger, **self._model_kwargs) dcrnn_model = STGCN(self.num_nodes, self.input_dim, self.seq_len, self.horizon) self.dcrnn_model = dcrnn_model.cuda() if torch.cuda.is_available( ) else dcrnn_model self._logger.info("Model created") self._epoch_num = self._train_kwargs.get('epoch', 0) # if self._epoch_num > 0: #事实上self._epoch_num的预设值确实为0 # self.load_model() self.data_type = data_type self.LOAD_INITIAL = LOAD_INITIAL if LOAD_INITIAL: self.load_lfx() self.A_wave = torch.from_numpy(get_normalized_adj(adj_mx)).to(device)
def __init__(self, catalog, override=False, schema='public'): self.conn = SQLConnection('xmm', schema) self.conn.autocommit = True self.working_format = 'votable' self.logger = get_logger('ICFx', log_filename='/tmp/icfx.log') self.logger.setLevel(logging.DEBUG) self.set_catalog(catalog) self.override = override self.source_catalogs = self.conn.execute_set(""" select distinct source_catalog from fields;""", quiet=False) self.source_catalogs = [x[0] for x in self.source_catalogs] sqllist.load_defaults() sqllist.GLOBALS['c'] = self.catalog.CATALOG
def __init__(self, total_obu_num): super().__init__() self.ID_GL = get_ID_gl(time.time()) self.total_num = total_obu_num self.logger = get_logger("GL") # ========= 生成属性 ========= self.delta_i_list = list() self.R = 0 self.W = 0 # ========= 信号量定义 ======= self.sem_delta_i_receive = Semaphore(0) self.sem_msg_rsp_receive = Semaphore(0) # ========= 网络设置 ======== self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) # ========= 回包 ========= self.rsp_msg = None
viz = Visualizations(device) ################################################################## #Load checkpoint and evaluate the model if args.load is not None: utils.get_ckpt_model(ckpt_path, model, device) exit() ################################################################## # Training log_path = "logs/" + file_name + "_" + str(experimentID) + ".log" if not os.path.exists("logs/"): utils.makedirs("logs/") logger = utils.get_logger(logpath=log_path, filepath=os.path.abspath(__file__)) logger.info(input_command) optimizer = optim.Adamax(model.parameters(), lr=args.lr) num_batches = data_obj["n_train_batches"] for itr in range(1, num_batches * (args.niters + 1)): optimizer.zero_grad() utils.update_learning_rate(optimizer, decay_rate=0.999, lowest=args.lr / 10) wait_until_kl_inc = 10 if itr // num_batches < wait_until_kl_inc: kl_coef = 0.
def __init__(self, is_training=False, **kwargs): self._kwargs = kwargs self._alg = kwargs.get('alg') self._data_kwargs = kwargs.get('data') self._train_kwargs = kwargs.get('train') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._base_dir = kwargs.get('base_dir') self._epochs = self._train_kwargs.get('epochs') # logging. self._log_dir = get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._writer = tf.summary.FileWriter(self._log_dir) self._logger.info(kwargs) self._mon_ratio = float(self._kwargs.get('mon_ratio')) # Model's args self._input_dim = int(self._model_kwargs.get('input_dim')) self._nodes = int(self._model_kwargs.get('num_nodes')) self._drop_out = float(self._train_kwargs.get('dropout')) self.batch_size = int(self._data_kwargs['batch_size']) # Test's args self._flow_selection = self._test_kwargs.get('flow_selection') self._run_times = self._test_kwargs.get('run_times') # Data preparation self._day_size = self._data_kwargs.get('day_size') self._data = utils.load_dataset_gatlstm( num_nodes=self._model_kwargs.get('num_nodes'), input_dim=self._model_kwargs.get('input_dim'), mon_ratio=self._mon_ratio, scaler_type=self._kwargs.get('scaler'), is_training=is_training, **self._data_kwargs) for k, v in self._data.items(): if hasattr(v, 'shape'): self._logger.info((k, v.shape)) # Build models. scaler = self._data['scaler'] if is_training: self.model = GATLSTMModel(scaler=scaler, batch_size=self.batch_size, **self._model_kwargs) else: self.model = GATLSTMModel(scaler=scaler, batch_size=1, **self._model_kwargs) # Learning rate. max_to_keep = self._train_kwargs.get('max_to_keep', 100) self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep) self.summary_writer = tf.summary.FileWriter(self._log_dir) # Log model statistics. total_trainable_parameter = utils.get_total_trainable_parameter_size() self._logger.info('Total number of trainable parameters: {:d}'.format( total_trainable_parameter)) for var in tf.global_variables(): self._logger.debug('{}, {}'.format(var.name, var.get_shape()))
def __init__(self, args, adj_mx, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._model_kwargs = kwargs.get('model') self._train_kwargs = kwargs.get('train') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, kwargs['name'] + '_info.log', level=log_level) self._writer = tf.summary.FileWriter(self._log_dir) self._logger.info(kwargs) # Data preparation if self._data_kwargs.get('data_type') == 'npz': self._data = utils.load_dataset(**self._data_kwargs) elif self._data_kwargs.get('data_type') == 'csv': self._data = utils.load_dataset_from_csv(**self._data_kwargs) for k, v in self._data.items(): if hasattr(v, 'shape'): self._logger.info((k, v.shape)) # Build models. scaler = self._data['scaler'] with tf.name_scope('Train'): with tf.variable_scope('DCRNN', reuse=False): self._train_model = DCRNNModel( args=args, is_training=True, scaler=scaler, batch_size=self._data_kwargs['batch_size'], adj_mx=adj_mx, **self._model_kwargs) with tf.name_scope('Test'): with tf.variable_scope('DCRNN', reuse=True): self._test_model = DCRNNModel( args=args, is_training=False, scaler=scaler, batch_size=self._data_kwargs['test_batch_size'], adj_mx=adj_mx, **self._model_kwargs) # Learning rate. self._lr = tf.get_variable('learning_rate', shape=(), initializer=tf.constant_initializer(0.01), trainable=False) self._new_lr = tf.placeholder(tf.float32, shape=(), name='new_learning_rate') self._lr_update = tf.assign(self._lr, self._new_lr, name='lr_update') # Configure optimizer optimizer_name = self._train_kwargs.get('optimizer', 'adam').lower() epsilon = float(self._train_kwargs.get('epsilon', 1e-3)) optimizer = tf.train.AdamOptimizer(self._lr, epsilon=epsilon) if optimizer_name == 'sgd': optimizer = tf.train.GradientDescentOptimizer(self._lr, ) elif optimizer_name == 'amsgrad': optimizer = AMSGrad(self._lr, epsilon=epsilon) # Calculate loss output_dim = self._model_kwargs.get('output_dim') preds = self._train_model.outputs labels = self._train_model.labels[..., :output_dim] null_val = 0. self._loss_fn = masked_mae_loss(scaler, null_val) self._mape_fn = masked_mape(scaler, null_val) self._rmse_fn = masked_rmse_loss(scaler, null_val) self._train_loss = self._loss_fn(preds=preds, labels=labels) tvars = tf.trainable_variables() grads = tf.gradients(self._train_loss, tvars) max_grad_norm = kwargs['train'].get('max_grad_norm', 1.) grads, _ = tf.clip_by_global_norm(grads, max_grad_norm) global_step = tf.train.get_or_create_global_step() self._train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=global_step, name='train_op') max_to_keep = self._train_kwargs.get('max_to_keep', 100) self._epoch = 0 self._saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep) # Log model statistics. total_trainable_parameter = utils.get_total_trainable_parameter_size() self._logger.info('Total number of trainable parameters: {:d}'.format( total_trainable_parameter)) for var in tf.global_variables(): self._logger.debug('{}, {}'.format(var.name, var.get_shape()))
def __init__(self, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._train_kwargs = kwargs.get('train') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._alg_name = self._kwargs.get('alg') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._logger.info(kwargs) # Data's args self._day_size = self._data_kwargs.get('day_size') # Model's Args self._model_type = self._model_kwargs.get('model_type') self._rnn_units = self._model_kwargs.get('rnn_units') self._seq_len = self._model_kwargs.get('seq_len') self._horizon = self._model_kwargs.get('horizon') self._input_dim = self._model_kwargs.get('input_dim') self._input_shape = (self._seq_len, self._input_dim) self._output_dim = self._model_kwargs.get('output_dim') self._nodes = self._model_kwargs.get('num_nodes') self._n_rnn_layers = self._model_kwargs.get('n_rnn_layers') # Train's args self._drop_out = self._train_kwargs.get('dropout') self._epochs = self._train_kwargs.get('epochs') self._batch_size = self._data_kwargs.get('batch_size') # Test's args self._run_times = self._test_kwargs.get('run_times') self._flow_selection = self._test_kwargs.get('flow_selection') self._test_size = self._test_kwargs.get('test_size') self._results_path = self._test_kwargs.get('results_path') self._mon_ratio = self._kwargs.get('mon_ratio') # Load data if self._model_type == 'lstm': self._data = utils.load_dataset_lstm(seq_len=self._seq_len, horizon=self._horizon, input_dim=self._input_dim, mon_ratio=self._mon_ratio, test_size=self._test_size, **self._data_kwargs) elif self._model_type == 'ed' or self._model_type == 'encoder_decoder': self._data = utils.load_dataset_lstm_ed(seq_len=self._seq_len, horizon=self._horizon, input_dim=self._input_dim, mon_ratio=self._mon_ratio, test_size=self._test_size, **self._data_kwargs) else: raise RuntimeError("Model must be lstm or encoder_decoder") for k, v in self._data.items(): if hasattr(v, 'shape'): self._logger.info((k, v.shape)) # Model self.callbacks_list = [] self._checkpoints = ModelCheckpoint(self._log_dir + "best_model.hdf5", monitor='val_loss', verbose=1, save_best_only=True, mode='auto', period=1) self.callbacks_list = [self._checkpoints] self._earlystop = EarlyStopping( monitor='val_loss', patience=self._train_kwargs.get('patience'), verbose=1, mode='auto') self.callbacks_list.append(self._earlystop) self._time_callback = TimeHistory() self.callbacks_list.append(self._time_callback) self.model = None
"""Various pre-processing steps that are applied before converting to a spacy doc.""" import shutil from os.path import exists, join from typing import Dict import numpy as np import pandas import re import wget from abbreviations import schwartz_hearst import constants from lib.utils import get_logger logger = get_logger("Preprocessing") def load_unicode_mappings(datapath: str) -> Dict[int, str]: """ Load mappings between unicode and ascii characters. datapath: path to data directory """ mappings = {} filename = "entities.dat" if not exists(join(datapath, filename)): url = "https://structuredabstracts.nlm.nih.gov/Downloads/Structured-Abstracts-Labels-102615.txt" logger.debug( "File with unicode-to-ascii mappings not found. Downloading from %s.", url) filename = wget.download(url)
def __init__(self, is_training=True, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._train_kwargs = kwargs.get('train') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._alg_name = self._kwargs.get('alg') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._logger.info(kwargs) self._mon_ratio = self._kwargs.get('mon_ratio') # Data's args self._day_size = self._data_kwargs.get('day_size') # Model's Args self._hidden = self._model_kwargs.get('rnn_units') self._seq_len = self._model_kwargs.get('seq_len') self._horizon = self._model_kwargs.get('horizon') self._input_dim = self._model_kwargs.get('input_dim') self._input_shape = (self._seq_len, self._input_dim) self._output_dim = self._model_kwargs.get('output_dim') self._nodes = self._model_kwargs.get('num_nodes') self._r = self._model_kwargs.get('r') # Train's args self._drop_out = self._train_kwargs.get('dropout') self._epochs = self._train_kwargs.get('epochs') self._batch_size = self._data_kwargs.get('batch_size') # Test's args self._run_times = self._test_kwargs.get('run_times') self._flow_selection = self._test_kwargs.get('flow_selection') self._lamda = [] self._lamda.append(self._test_kwargs.get('lamda_0')) self._lamda.append(self._test_kwargs.get('lamda_1')) self._lamda.append(self._test_kwargs.get('lamda_2')) # Load data self._data = utils.load_dataset_fwbw_lstm_ed(seq_len=self._seq_len, horizon=self._horizon, input_dim=self._input_dim, mon_ratio=self._mon_ratio, scaler_type=self._kwargs.get('scaler'), **self._data_kwargs) for k, v in self._data.items(): if hasattr(v, 'shape'): self._logger.info((k, v.shape)) # Model self.model = self.construct_fwbw_lstm_ed(is_training=is_training) self.callbacks_list = [] self._checkpoints = ModelCheckpoint( self._log_dir + "best_model.hdf5", monitor='val_loss', verbose=1, save_best_only=True, mode='auto', period=1) self.callbacks_list = [self._checkpoints] self._earlystop = EarlyStopping(monitor='val_loss', patience=self._train_kwargs.get('patience'), verbose=1, mode='auto') self.callbacks_list.append(self._earlystop) self._time_callback = TimeHistory() self.callbacks_list.append(self._time_callback)
def __init__(self, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._train_kwargs = kwargs.get('train') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._alg_name = self._kwargs.get('alg') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._logger.info(kwargs) # Data's args self._day_size = self._data_kwargs.get('day_size') # Model's Args self._seq_len = self._model_kwargs.get('seq_len') self._horizon = self._model_kwargs.get('horizon') self._output_dim = self._model_kwargs.get('output_dim') self._nodes = self._model_kwargs.get('num_nodes') self._wide = self._model_kwargs.get('wide') self._high = self._model_kwargs.get('high') self._channel = self._model_kwargs.get('channel') self._filters = self._model_kwargs.get('filters') self._kernel_size = self._model_kwargs.get('kernel_size') self._strides = self._model_kwargs.get('strides') self._input_shape = (self._seq_len, self._wide, self._high, self._channel) # Train's args self._conv_dropout = self._train_kwargs.get('conv_dropout') self._rnn_dropout = self._train_kwargs.get('rnn_dropout') self._epochs = self._train_kwargs.get('epochs') self._batch_size = self._data_kwargs.get('batch_size') # Test's args self._run_times = self._test_kwargs.get('run_times') self._flow_selection = self._test_kwargs.get('flow_selection') self._test_size = self._test_kwargs.get('test_size') self._results_path = self._test_kwargs.get('results_path') self._lamda = [] self._lamda.append(self._test_kwargs.get('lamda_0')) self._lamda.append(self._test_kwargs.get('lamda_1')) self._lamda.append(self._test_kwargs.get('lamda_2')) self._mon_ratio = self._kwargs.get('mon_ratio') # Load data self._data = utils.load_dataset_conv_lstm(seq_len=self._seq_len, wide=self._wide, high=self._high, channel=self._channel, mon_ratio=self._mon_ratio, test_size=self._test_size, **self._data_kwargs) for k, v in self._data.items(): if hasattr(v, 'shape'): self._logger.info((k, v.shape)) # Model self.model = None self.callbacks_list = [] self._checkpoints = ModelCheckpoint(self._log_dir + "best_model.hdf5", monitor='val_loss', verbose=1, save_best_only=True, mode='auto', period=1) self.callbacks_list = [self._checkpoints] self._earlystop = EarlyStopping( monitor='val_loss', patience=self._train_kwargs.get('patience'), verbose=1, mode='auto') self.callbacks_list.append(self._earlystop) self._time_callback = TimeHistory() self.callbacks_list.append(self._time_callback)
def train_toy(toy, load=True, nb_step_dual=300, nb_steps=15, folder="", l1=1., nb_epoch=20000, pre_heating_epochs=10, nb_flow=3, cond_type="Coupling", emb_net=[150, 150, 150]): logger = utils.get_logger(logpath=os.path.join(folder, toy, 'logs'), filepath=os.path.abspath(__file__)) logger.info("Creating model...") device = "cpu" if not (torch.cuda.is_available()) else "cuda:0" nb_samp = 100 batch_size = 100 x_test = torch.tensor(toy_data.inf_train_gen(toy, batch_size=1000)).to(device) x = torch.tensor(toy_data.inf_train_gen(toy, batch_size=1000)).to(device) dim = x.shape[1] norm_type = "Affine" save_name = norm_type + str(emb_net) + str(nb_flow) solver = "CCParallel" int_net = [150, 150, 150] conditioner_type = cond_types[cond_type] conditioner_args = { "in_size": dim, "hidden": emb_net[:-1], "out_size": emb_net[-1] } if conditioner_type is DAGConditioner: conditioner_args['l1'] = l1 conditioner_args['gumble_T'] = .5 conditioner_args['nb_epoch_update'] = nb_step_dual conditioner_args["hot_encoding"] = True normalizer_type = norm_types[norm_type] if normalizer_type is MonotonicNormalizer: normalizer_args = { "integrand_net": int_net, "cond_size": emb_net[-1], "nb_steps": nb_steps, "solver": solver } else: normalizer_args = {} model = buildFCNormalizingFlow(nb_flow, conditioner_type, conditioner_args, normalizer_type, normalizer_args) opt = torch.optim.Adam(model.parameters(), 1e-4, weight_decay=1e-5) if load: logger.info("Loading model...") model.load_state_dict( torch.load(folder + toy + '/' + save_name + 'model.pt')) model.train() opt.load_state_dict( torch.load(folder + toy + '/' + save_name + 'ADAM.pt')) logger.info("Model loaded.") if True: for step in model.steps: step.conditioner.stoch_gate = True step.conditioner.noise_gate = False step.conditioner.gumble_T = .5 torch.autograd.set_detect_anomaly(True) for epoch in range(nb_epoch): loss_tot = 0 start = timer() for j in range(0, nb_samp, batch_size): cur_x = torch.tensor( toy_data.inf_train_gen(toy, batch_size=batch_size)).to(device) z, jac = model(cur_x) loss = model.loss(z, jac) loss_tot += loss.detach() if math.isnan(loss.item()): ll, z = model.compute_ll(cur_x) print(ll) print(z) print(ll.max(), z.max()) exit() opt.zero_grad() loss.backward(retain_graph=True) opt.step() model.step(epoch, loss_tot) end = timer() z, jac = model(x_test) ll = (model.z_log_density(z) + jac) ll_test = -ll.mean() dagness = max(model.DAGness()) logger.info( "epoch: {:d} - Train loss: {:4f} - Test loss: {:4f} - <<DAGness>>: {:4f} - Elapsed time per epoch {:4f} (seconds)" .format(epoch, loss_tot.item(), ll_test.item(), dagness, end - start)) if epoch % 100 == 0 and False: with torch.no_grad(): stoch_gate = model.getDag().stoch_gate noise_gate = model.getDag().noise_gate s_thresh = model.getDag().s_thresh model.getDag().stoch_gate = False model.getDag().noise_gate = False model.getDag().s_thresh = True for threshold in [.95, .1, .01, .0001, 1e-8]: model.set_h_threshold(threshold) # Valid loop z, jac = model(x_test) ll = (model.z_log_density(z) + jac) ll_test = -ll.mean().item() dagness = max(model.DAGness()).item() logger.info( "epoch: {:d} - Threshold: {:4f} - Valid log-likelihood: {:4f} - <<DAGness>>: {:4f}" .format(epoch, threshold, ll_test, dagness)) model.getDag().stoch_gate = stoch_gate model.getDag().noise_gate = noise_gate model.getDag().s_thresh = s_thresh model.set_h_threshold(0.) if epoch % 500 == 0: font = {'family': 'normal', 'weight': 'normal', 'size': 25} matplotlib.rc('font', **font) if toy in [ "2spirals-8gaussians", "4-2spirals-8gaussians", "8-2spirals-8gaussians", "2gaussians", "4gaussians", "2igaussians", "8gaussians" ] or True: def compute_ll(x): z, jac = model(x) ll = (model.z_log_density(z) + jac) return ll, z with torch.no_grad(): npts = 100 plt.figure(figsize=(12, 12)) gs = gridspec.GridSpec(2, 2, width_ratios=[3, 1], height_ratios=[3, 1]) ax = plt.subplot(gs[0]) qz_1, qz_2 = vf.plt_flow(compute_ll, ax, npts=npts, device=device) plt.subplot(gs[1]) plt.plot(qz_1, np.linspace(-4, 4, npts)) plt.ylabel('$x_2$', fontsize=25, rotation=-90, labelpad=20) plt.xticks([]) plt.subplot(gs[2]) plt.plot(np.linspace(-4, 4, npts), qz_2) plt.xlabel('$x_1$', fontsize=25) plt.yticks([]) plt.savefig("%s%s/flow_%s_%d.pdf" % (folder, toy, save_name, epoch)) torch.save(model.state_dict(), folder + toy + '/' + save_name + 'model.pt') torch.save(opt.state_dict(), folder + toy + '/' + save_name + 'ADAM.pt')
def run(args, kwargs): # ================================================================================================================== # SNAPSHOTS # ================================================================================================================== args.model_signature = str(datetime.datetime.now())[0:19].replace(' ', '_') args.model_signature = args.model_signature.replace(':', '_') snapshots_path = os.path.join(args.out_dir, 'vae_' + args.dataset + '_') snap_dir = snapshots_path + args.flow if args.flow != 'no_flow': snap_dir += '_' + 'num_flows_' + str(args.num_flows) if args.flow == 'orthogonal': snap_dir = snap_dir + '_num_vectors_' + str(args.num_ortho_vecs) elif args.flow == 'orthogonalH': snap_dir = snap_dir + '_num_householder_' + str(args.num_householder) elif args.flow == 'iaf': snap_dir = snap_dir + '_madehsize_' + str(args.made_h_size) elif args.flow == 'permutation': snap_dir = snap_dir + '_' + 'kernelsize_' + str(args.kernel_size) elif args.flow == 'mixed': snap_dir = snap_dir + '_' + 'num_householder_' + str(args.num_householder) elif args.flow == 'cnf_rank': snap_dir = snap_dir + '_rank_' + str(args.rank) + '_' + args.dims + '_num_blocks_' + str(args.num_blocks) elif 'cnf' in args.flow: snap_dir = snap_dir + '_' + args.dims + '_num_blocks_' + str(args.num_blocks) if args.retrain_encoder: snap_dir = snap_dir + '_retrain-encoder_' elif args.evaluate: snap_dir = snap_dir + '_evaluate_' snap_dir = snap_dir + '__' + args.model_signature + '/' args.snap_dir = snap_dir if not os.path.exists(snap_dir): os.makedirs(snap_dir) # logger utils.makedirs(args.snap_dir) logger = utils.get_logger(logpath=os.path.join(args.snap_dir, 'logs'), filepath=os.path.abspath(__file__)) logger.info(args) # SAVING torch.save(args, snap_dir + args.flow + '.config') # ================================================================================================================== # LOAD DATA # ================================================================================================================== train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs) if not args.evaluate: # ============================================================================================================== # SELECT MODEL # ============================================================================================================== # flow parameters and architecture choice are passed on to model through args if args.flow == 'no_flow': model = VAE.VAE(args) elif args.flow == 'planar': model = VAE.PlanarVAE(args) elif args.flow == 'iaf': model = VAE.IAFVAE(args) elif args.flow == 'orthogonal': model = VAE.OrthogonalSylvesterVAE(args) elif args.flow == 'householder': model = VAE.HouseholderSylvesterVAE(args) elif args.flow == 'triangular': model = VAE.TriangularSylvesterVAE(args) elif args.flow == 'cnf': model = CNFVAE.CNFVAE(args) elif args.flow == 'cnf_bias': model = CNFVAE.AmortizedBiasCNFVAE(args) elif args.flow == 'cnf_hyper': model = CNFVAE.HypernetCNFVAE(args) elif args.flow == 'cnf_lyper': model = CNFVAE.LypernetCNFVAE(args) elif args.flow == 'cnf_rank': model = CNFVAE.AmortizedLowRankCNFVAE(args) else: raise ValueError('Invalid flow choice') if args.retrain_encoder: logger.info(f"Initializing decoder from {args.model_path}") dec_model = torch.load(args.model_path) dec_sd = {} for k, v in dec_model.state_dict().items(): if 'p_x' in k: dec_sd[k] = v model.load_state_dict(dec_sd, strict=False) if args.cuda: logger.info("Model on GPU") model.cuda() logger.info(model) if args.retrain_encoder: parameters = [] logger.info('Optimizing over:') for name, param in model.named_parameters(): if 'p_x' not in name: logger.info(name) parameters.append(param) else: parameters = model.parameters() optimizer = optim.Adamax(parameters, lr=args.learning_rate, eps=1.e-7) # ================================================================================================================== # TRAINING # ================================================================================================================== train_loss = [] val_loss = [] # for early stopping best_loss = np.inf best_bpd = np.inf e = 0 epoch = 0 train_times = [] for epoch in range(1, args.epochs + 1): t_start = time.time() tr_loss = train(epoch, train_loader, model, optimizer, args, logger) train_loss.append(tr_loss) train_times.append(time.time() - t_start) logger.info('One training epoch took %.2f seconds' % (time.time() - t_start)) v_loss, v_bpd = evaluate(val_loader, model, args, logger, epoch=epoch) val_loss.append(v_loss) # early-stopping if v_loss < best_loss: e = 0 best_loss = v_loss if args.input_type != 'binary': best_bpd = v_bpd logger.info('->model saved<-') torch.save(model, snap_dir + args.flow + '.model') # torch.save(model, snap_dir + args.flow + '_' + args.architecture + '.model') elif (args.early_stopping_epochs > 0) and (epoch >= args.warmup): e += 1 if e > args.early_stopping_epochs: break if args.input_type == 'binary': logger.info( '--> Early stopping: {}/{} (BEST: loss {:.4f})\n'.format(e, args.early_stopping_epochs, best_loss) ) else: logger.info( '--> Early stopping: {}/{} (BEST: loss {:.4f}, bpd {:.4f})\n'. format(e, args.early_stopping_epochs, best_loss, best_bpd) ) if math.isnan(v_loss): raise ValueError('NaN encountered!') train_loss = np.hstack(train_loss) val_loss = np.array(val_loss) plot_training_curve(train_loss, val_loss, fname=snap_dir + '/training_curve_%s.pdf' % args.flow) # training time per epoch train_times = np.array(train_times) mean_train_time = np.mean(train_times) std_train_time = np.std(train_times, ddof=1) logger.info('Average train time per epoch: %.2f +/- %.2f' % (mean_train_time, std_train_time)) # ================================================================================================================== # EVALUATION # ================================================================================================================== logger.info(args) logger.info('Stopped after %d epochs' % epoch) logger.info('Average train time per epoch: %.2f +/- %.2f' % (mean_train_time, std_train_time)) final_model = torch.load(snap_dir + args.flow + '.model') validation_loss, validation_bpd = evaluate(val_loader, final_model, args, logger) else: validation_loss = "N/A" validation_bpd = "N/A" logger.info(f"Loading model from {args.model_path}") final_model = torch.load(args.model_path) test_loss, test_bpd = evaluate(test_loader, final_model, args, logger, testing=True) logger.info('FINAL EVALUATION ON VALIDATION SET. ELBO (VAL): {:.4f}'.format(validation_loss)) logger.info('FINAL EVALUATION ON TEST SET. NLL (TEST): {:.4f}'.format(test_loss)) if args.input_type != 'binary': logger.info('FINAL EVALUATION ON VALIDATION SET. ELBO (VAL) BPD : {:.4f}'.format(validation_bpd)) logger.info('FINAL EVALUATION ON TEST SET. NLL (TEST) BPD: {:.4f}'.format(test_bpd))
def train(dataset="MNIST", load=True, nb_step_dual=100, nb_steps=20, path="", l1=.1, nb_epoch=10000, b_size=100, int_net=[50, 50, 50], all_args=None, file_number=None, train=True, solver="CC", weight_decay=1e-5, learning_rate=1e-3, batch_per_optim_step=1, n_gpu=1, norm_type='Affine', nb_flow=[1], hot_encoding=True, prior_A_kernel=None, conditioner="DAG", emb_net=None): logger = utils.get_logger(logpath=os.path.join(path, 'logs'), filepath=os.path.abspath(__file__)) logger.info(str(all_args)) if load: file_number = "_" + file_number if file_number is not None else "" batch_size = b_size best_valid_loss = np.inf logger.info("Loading data...") train_loader, valid_loader, test_loader = load_data(dataset, batch_size) if len(dataset) == 6 and dataset[:5] == 'MNIST': dataset = "MNIST" alpha = 1e-6 if dataset == "MNIST" else .05 logger.info("Data loaded.") master_device = "cuda:0" if torch.cuda.is_available() else "cpu" # ----------------------- Model Definition ------------------- # logger.info("Creating model...") if norm_type == 'Affine': normalizer_type = AffineNormalizer normalizer_args = {} else: normalizer_type = MonotonicNormalizer normalizer_args = { "integrand_net": int_net, "nb_steps": 15, "solver": solver } if conditioner == "DAG": conditioner_type = DAGConditioner if dataset == "MNIST": inner_model = buildMNISTNormalizingFlow( nb_flow, normalizer_type, normalizer_args, l1, nb_epoch_update=nb_step_dual, hot_encoding=hot_encoding, prior_kernel=prior_A_kernel) elif dataset == "CIFAR10": inner_model = buildCIFAR10NormalizingFlow( nb_flow, normalizer_type, normalizer_args, l1, nb_epoch_update=nb_step_dual, hot_encoding=hot_encoding) else: logger.info("Wrong dataset name. Training aborted.") exit() else: dim = 28**2 if dataset == "MNIST" else 32 * 32 * 3 conditioner_type = cond_types[conditioner] conditioner_args = { "in_size": dim, "hidden": emb_net[:-1], "out_size": emb_net[-1] } if norm_type == 'Monotonic': normalizer_args["cond_size"] = emb_net[-1] inner_model = buildFCNormalizingFlow(nb_flow[0], conditioner_type, conditioner_args, normalizer_type, normalizer_args) model = nn.DataParallel(inner_model, device_ids=list(range(n_gpu))).to(master_device) logger.info(str(model)) pytorch_total_params = sum(p.numel() for p in model.parameters()) logger.info("Number of parameters: %d" % pytorch_total_params) opt = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) if load: logger.info("Loading model...") model.load_state_dict( torch.load(path + '/model%s.pt' % file_number, map_location={"cuda:0": master_device})) model.train() opt.load_state_dict( torch.load(path + '/ADAM%s.pt' % file_number, map_location={"cuda:0": master_device})) if master_device != "cpu": for state in opt.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): state[k] = v.cuda() logger.info("...Model built.") logger.info("Training starts:") if load: for conditioner in model.module.getConditioners(): conditioner.alpha = conditioner.getAlpha() # ----------------------- Main Loop ------------------------- # for epoch in range(nb_epoch): ll_tot = 0 start = timer() if train: model.to(master_device) # ----------------------- Training Loop ------------------------- # for batch_idx, (cur_x, target) in enumerate(train_loader): cur_x = cur_x.view(batch_size, -1).float().to(master_device) for normalizer in model.module.getNormalizers(): if type(normalizer) is MonotonicNormalizer: normalizer.nb_steps = nb_steps + torch.randint( 0, 10, [1])[0].item() z, jac = model(cur_x) loss = model.module.loss(z, jac) / (batch_per_optim_step * n_gpu) if math.isnan(loss.item()): print("Error Nan in loss") print("Dagness:", model.module.DAGness()) exit() ll_tot += loss.detach() if batch_idx % batch_per_optim_step == 0: opt.zero_grad() loss.backward(retain_graph=True) if (batch_idx + 1) % batch_per_optim_step == 0: opt.step() with torch.no_grad(): print("Dagness:", model.module.DAGness()) ll_tot /= (batch_idx + 1) torch.cuda.empty_cache() model.module.step(epoch, ll_tot) else: ll_tot = 0. # ----------------------- Valid Loop ------------------------- # ll_test = 0. bpp_test = 0. model.to(master_device) with torch.no_grad(): for normalizer in model.module.getNormalizers(): if type(normalizer) is MonotonicNormalizer: normalizer.nb_steps = 150 for batch_idx, (cur_x, target) in enumerate(valid_loader): cur_x = cur_x.view(batch_size, -1).float().to(master_device) z, jac = model(cur_x) ll = (model.module.z_log_density(z) + jac) ll_test += ll.mean().item() bpp_test += compute_bpp( ll, cur_x.view(batch_size, -1).float().to(master_device), alpha).mean().item() ll_test /= batch_idx + 1 bpp_test /= batch_idx + 1 end = timer() dagness = max(model.module.DAGness()) logger.info( "epoch: {:d} - Train loss: {:4f} - Valid log-likelihood: {:4f} - Valid BPP {:4f} - <<DAGness>>: {:4f} " "- Elapsed time per epoch {:4f} (seconds)".format( epoch, ll_tot, ll_test, bpp_test, dagness, end - start)) if model.module.isInvertible() and -ll_test < best_valid_loss: logger.info("------- New best validation loss --------") torch.save(model.state_dict(), path + '/best_model.pt') best_valid_loss = -ll_test # Valid loop ll_test = 0. for batch_idx, (cur_x, target) in enumerate(test_loader): z, jac = model( cur_x.view(batch_size, -1).float().to(master_device)) ll = (model.module.z_log_density(z) + jac) ll_test += ll.mean().item() bpp_test += compute_bpp( ll, cur_x.view(batch_size, -1).float().to(master_device), alpha).mean().item() ll_test /= batch_idx + 1 bpp_test /= batch_idx + 1 logger.info( "epoch: {:d} - Test log-likelihood: {:4f} - Test BPP {:4f} - <<DAGness>>: {:4f}" .format(epoch, ll_test, bpp_test, dagness)) if epoch % 10 == 0 and conditioner_type is DAGConditioner: stoch_gate, noise_gate, s_thresh = [], [], [] for conditioner in model.module.getConditioners(): stoch_gate.append(conditioner.stoch_gate) noise_gate.append(conditioner.noise_gate) s_thresh.append(conditioner.s_thresh) conditioner.stoch_gate = False conditioner.noise_gate = False conditioner.s_thresh = True for threshold in [.95, .5, .1, .01, .0001]: for conditioner in model.module.getConditioners(): conditioner.h_thresh = threshold # Valid loop ll_test = 0. bpp_test = 0. for batch_idx, (cur_x, target) in enumerate(valid_loader): cur_x = cur_x.view(batch_size, -1).float().to(master_device) z, jac = model(cur_x) ll = (model.module.z_log_density(z) + jac) ll_test += ll.mean().item() bpp_test += compute_bpp( ll, cur_x.view(batch_size, -1).float().to(master_device), alpha).mean().item() ll_test /= batch_idx + 1 bpp_test /= batch_idx + 1 dagness = max(model.module.DAGness()) logger.info( "epoch: {:d} - Threshold: {:4f} - Valid log-likelihood: {:4f} - Valid BPP {:4f} - <<DAGness>>: {:4f}" .format(epoch, threshold, ll_test, bpp_test, dagness)) for i, conditioner in enumerate( model.module.getConditioners()): conditioner.h_thresh = 0. conditioner.stoch_gate = stoch_gate[i] conditioner.noise_gate = noise_gate[i] conditioner.s_thresh = s_thresh[i] in_s = 784 if dataset == "MNIST" else 3 * 32 * 32 a_tmp = model.module.getConditioners()[0].soft_thresholded_A()[ 0, :] a_tmp = a_tmp.view( 28, 28).cpu().numpy() if dataset == "MNIST" else a_tmp.view( 3, 32, 32).cpu().numpy() fig, ax = plt.subplots() mat = ax.matshow(a_tmp) plt.colorbar(mat) current_cmap = matplotlib.cm.get_cmap() current_cmap.set_bad(color='red') mat.set_clim(0, 1.) def update(i): A = model.module.getConditioners()[0].soft_thresholded_A()[ i, :].cpu().numpy() A[i] = np.nan if dataset == "MNIST": A = A.reshape(28, 28) elif dataset == "CIFAR10": A = A.reshape(3, 32, 32) mat.set_data(A) return mat # Set up formatting for the movie files Writer = animation.writers['ffmpeg'] writer = Writer(fps=15, metadata=dict(artist='Me'), bitrate=1800) ani = animation.FuncAnimation(fig, update, range(in_s), interval=100, save_count=0) ani.save(path + '/A_epoch_%d.mp4' % epoch, writer=writer) deg_out = ( model.module.getConditioners()[0].soft_thresholded_A() > 0.).sum(0).cpu().numpy() deg_in = ( model.module.getConditioners()[0].soft_thresholded_A() > 0.).sum(1).cpu().numpy() fig, ax = plt.subplots(1, 2, figsize=(12, 6)) if dataset == "MNIST": shape = (28, 28) elif dataset == "CIFAR10": shape = (3, 32, 32) res0 = ax[0].matshow(np.log(deg_in).reshape(shape)) ax[0].set(title="In degrees") fig.colorbar(res0, ax=ax[0]) res1 = ax[1].matshow(np.log(deg_out.reshape(shape))) ax[1].set(title="Out degrees") fig.colorbar(res1, ax=ax[1]) plt.savefig(path + '/A_degrees_epoch_%d.png' % epoch) if model.module.isInvertible(): with torch.no_grad(): n_images = 16 in_s = 28**2 for T in [.1, .25, .5, .75, 1.]: z = torch.randn(n_images, in_s).to(device=master_device) * T x = model.module.invert(z) print((z - model(x)[0]).abs().mean()) grid_img = torchvision.utils.make_grid(x.view( n_images, 1, 28, 28), nrow=4) torchvision.utils.save_image( grid_img, path + '/images_%d_%f.png' % (epoch, T)) if epoch % nb_step_dual == 0: logger.info("Saving model N°%d" % epoch) torch.save(model.state_dict(), path + '/model_%d.pt' % epoch) torch.save(opt.state_dict(), path + '/ADAM_%d.pt' % epoch) torch.save(model.state_dict(), path + '/model.pt') torch.save(opt.state_dict(), path + '/ADAM.pt') torch.cuda.empty_cache()
def train(dataset="POWER", load=True, nb_step_dual=100, nb_steps=20, path="", l1=.1, nb_epoch=10000, int_net=[200, 200, 200], emb_net=[200, 200, 200], b_size=100, all_args=None, file_number=None, train=True, solver="CC", nb_flow=1, weight_decay=1e-5, learning_rate=1e-3, cond_type='DAG', norm_type='affine'): logger = utils.get_logger(logpath=os.path.join(path, 'logs'), filepath=os.path.abspath(__file__)) logger.info(str(all_args)) logger.info("Creating model...") device = "cpu" if not (torch.cuda.is_available()) else "cuda:0" if load: #train = False file_number = "_" + file_number if file_number is not None else "" batch_size = b_size logger.info("Loading data...") data = load_data(dataset) data.trn.x = torch.from_numpy(data.trn.x).to(device) data.val.x = torch.from_numpy(data.val.x).to(device) data.tst.x = torch.from_numpy(data.tst.x).to(device) logger.info("Data loaded.") dim = data.trn.x.shape[1] conditioner_type = cond_types[cond_type] conditioner_args = { "in_size": dim, "hidden": emb_net[:-1], "out_size": emb_net[-1] } if conditioner_type is DAGConditioner: conditioner_args['l1'] = l1 conditioner_args['gumble_T'] = .5 conditioner_args['nb_epoch_update'] = nb_step_dual conditioner_args["hot_encoding"] = True normalizer_type = norm_types[norm_type] if normalizer_type is MonotonicNormalizer: normalizer_args = { "integrand_net": int_net, "cond_size": emb_net[-1], "nb_steps": nb_steps, "solver": solver } else: normalizer_args = {} model = buildFCNormalizingFlow(nb_flow, conditioner_type, conditioner_args, normalizer_type, normalizer_args) best_valid_loss = np.inf opt = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) if load: logger.info("Loading model...") model.load_state_dict( torch.load(path + '/model%s.pt' % file_number, map_location={"cuda:0": device})) model.train() if os.path.isfile(path + '/ADAM%s.pt'): opt.load_state_dict( torch.load(path + '/ADAM%s.pt' % file_number, map_location={"cuda:0": device})) if device != "cpu": for state in opt.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): state[k] = v.cuda() #x = data.trn.x[:20] #print(x, model(x)) #exit() for epoch in range(nb_epoch): ll_tot = 0 start = timer() # Update constraints if conditioner_type is DAGConditioner: with torch.no_grad(): for conditioner in model.getConditioners(): conditioner.constrainA(zero_threshold=0.) # Training loop model.to(device) if train: for i, cur_x in enumerate( batch_iter(data.trn.x, shuffle=True, batch_size=batch_size)): if normalizer_type is MonotonicNormalizer: for normalizer in model.getNormalizers(): normalizer.nb_steps = nb_steps + torch.randint( 0, 10, [1])[0].item() z, jac = model(cur_x) #print(z.mean(), jac.mean()) loss = model.loss(z, jac) if math.isnan(loss.item()) or math.isinf(loss.abs().item()): torch.save(model.state_dict(), path + '/NANmodel.pt') print("Error NAN in loss") exit() ll_tot += loss.detach() opt.zero_grad() loss.backward(retain_graph=True) opt.step() ll_tot /= i + 1 model.step(epoch, ll_tot) else: ll_tot = 0. # Valid loop ll_test = 0. with torch.no_grad(): if normalizer_type is MonotonicNormalizer: for normalizer in model.getNormalizers(): normalizer.nb_steps = nb_steps + 20 for i, cur_x in enumerate( batch_iter(data.val.x, shuffle=True, batch_size=batch_size)): z, jac = model(cur_x) ll = (model.z_log_density(z) + jac) ll_test += ll.mean().item() ll_test /= i + 1 end = timer() dagness = max(model.DAGness()) logger.info( "epoch: {:d} - Train loss: {:4f} - Valid log-likelihood: {:4f} - <<DAGness>>: {:4f} - Elapsed time per epoch {:4f} (seconds)" .format(epoch, ll_tot.item(), ll_test, dagness, end - start)) if dagness < 1e-20 and -ll_test < best_valid_loss: logger.info("------- New best validation loss --------") torch.save(model.state_dict(), path + '/best_model.pt') best_valid_loss = -ll_test # Valid loop ll_test = 0. for i, cur_x in enumerate( batch_iter(data.tst.x, shuffle=True, batch_size=batch_size)): z, jac = model(cur_x) ll = (model.z_log_density(z) + jac) ll_test += ll.mean().item() ll_test /= i + 1 logger.info( "epoch: {:d} - Test log-likelihood: {:4f} - <<DAGness>>: {:4f}" .format(epoch, ll_test, dagness)) if epoch % 10 == 0 and conditioner_type is DAGConditioner: stoch_gate, noise_gate, s_thresh = [], [], [] for conditioner in model.getConditioners(): stoch_gate.append(conditioner.stoch_gate) noise_gate.append(conditioner.noise_gate) s_thresh.append(conditioner.s_thresh) conditioner.stoch_gate = False conditioner.noise_gate = False conditioner.s_thresh = True for threshold in [.95, .5, .1, .01, .0001]: for conditioner in model.getConditioners(): conditioner.h_thresh = threshold # Valid loop ll_test = 0. for i, cur_x in enumerate( batch_iter(data.val.x, shuffle=True, batch_size=batch_size)): z, jac = model(cur_x) ll = (model.z_log_density(z) + jac) ll_test += ll.mean().item() ll_test /= i dagness = max(model.DAGness()) logger.info( "epoch: {:d} - Threshold: {:4f} - Valid log-likelihood: {:4f} - <<DAGness>>: {:4f}" .format(epoch, threshold, ll_test, dagness)) for i, conditioner in enumerate(model.getConditioners()): conditioner.h_thresh = threshold conditioner.stoch_gate = stoch_gate[i] conditioner.noise_gate = noise_gate[i] conditioner.s_thresh = s_thresh[i] torch.save(model.state_dict(), path + '/model_%d.pt' % epoch) torch.save(opt.state_dict(), path + '/ADAM_%d.pt' % epoch) if dataset == "proteins" and conditioner_type is DAGConditioner: torch.save( model.getConditioners[0].soft_thresholded_A().detach().cpu( ), path + '/A_%d.pt' % epoch) torch.save(model.state_dict(), path + '/model.pt') torch.save(opt.state_dict(), path + '/ADAM.pt')
def main(args): cfg = read_cfg_file(args.config_filename) log_dir = _get_log_dir(cfg) log_level = cfg.get('log_level', 'INFO') logger = utils.get_logger(log_dir, __name__, 'info.log', level=log_level) device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') # all edge_index in same dataset is same # edge_index = adjacency_to_edge_index(adj_mx) # alreay added self-loop logger.info(cfg) # batch_size = cfg['data']['batch_size'] # test_batch_size = cfg['data']['test_batch_size'] # edge_index = utils.load_pickle(cfg['data']['edge_index_pkl_filename']) hz = cfg['data'].get('name', 'nothz') == 'hz' adj_mx_list = [] graph_pkl_filename = cfg['data']['graph_pkl_filename'] if not isinstance(graph_pkl_filename, list): graph_pkl_filename = [graph_pkl_filename] src = [] dst = [] for g in graph_pkl_filename: if hz: adj_mx = utils.load_graph_data_hz(g) else: _, _, adj_mx = utils.load_graph_data(g) for i in range(len(adj_mx)): adj_mx[i, i] = 0 adj_mx_list.append(adj_mx) adj_mx = np.stack(adj_mx_list, axis=-1) if cfg['model'].get('norm', False): print('row normalization') adj_mx = adj_mx / (adj_mx.sum(axis=0) + 1e-18) src, dst = adj_mx.sum(axis=-1).nonzero() edge_index = torch.tensor([src, dst], dtype=torch.long, device=device) edge_attr = torch.tensor(adj_mx[adj_mx.sum(axis=-1) != 0], dtype=torch.float, device=device) output_dim = cfg['model']['output_dim'] for i in range(adj_mx.shape[-1]): logger.info(adj_mx[..., i]) # print(adj_mx.shape) (207, 207) if hz: dataset = utils.load_dataset_hz(**cfg['data'], scaler_axis=(0, 1, 2, 3)) else: dataset = utils.load_dataset(**cfg['data']) for k, v in dataset.items(): if hasattr(v, 'shape'): logger.info((k, v.shape)) model = Net(cfg).to(device) model.load_state_dict(torch.load(cfg['model']['save_path']), strict=False) evaluate(model=model, dataset=dataset, dataset_type='test', edge_index=edge_index, edge_attr=edge_attr, device=device, output_dim=output_dim, logger=logger, cfg=cfg, format_result=True)
def main(): # os.system('shutdown -c') # cancel previous shutdown command if write_log: utils.makedirs(args.save) logger = utils.get_logger(logpath=os.path.join(args.save, 'logs'), filepath=os.path.abspath(__file__)) logger.info(args) args_file_path = os.path.join(args.save, 'args.yaml') with open(args_file_path, 'w') as f: yaml.dump(vars(args), f, default_flow_style=False) if args.distributed: if write_log: logger.info('Distributed initializing process group') torch.cuda.set_device(args.local_rank) distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=dist_utils.env_world_size(), rank=env_rank()) assert (dist_utils.env_world_size() == distributed.get_world_size()) if write_log: logger.info("Distributed: success (%d/%d)" % (args.local_rank, distributed.get_world_size())) device = torch.device("cuda:%d" % torch.cuda.current_device() if torch.cuda.is_available() else "cpu") else: device = torch.cuda.current_device() # # import pdb; pdb.set_trace() cvt = lambda x: x.type(torch.float32).to(device, non_blocking=True) # load dataset train_loader, test_loader, data_shape = get_dataset(args) trainlog = os.path.join(args.save, 'training.csv') testlog = os.path.join(args.save, 'test.csv') traincolumns = ['itr', 'wall', 'itr_time', 'loss', 'bpd', 'fe', 'total_time', 'grad_norm'] testcolumns = ['wall', 'epoch', 'eval_time', 'bpd', 'fe', 'total_time', 'transport_cost'] # build model regularization_fns, regularization_coeffs = create_regularization_fns(args) model = create_model(args, data_shape, regularization_fns).cuda() if args.distributed: model = dist_utils.DDP(model, device_ids=[args.local_rank], output_device=args.local_rank) traincolumns = append_regularization_keys_header(traincolumns, regularization_fns) if not args.resume and write_log: with open(trainlog, 'w') as f: csvlogger = csv.DictWriter(f, traincolumns) csvlogger.writeheader() with open(testlog, 'w') as f: csvlogger = csv.DictWriter(f, testcolumns) csvlogger.writeheader() set_cnf_options(args, model) if write_log: logger.info(model) if write_log: logger.info("Number of trainable parameters: {}".format(count_parameters(model))) if write_log: logger.info('Iters per train epoch: {}'.format(len(train_loader))) if write_log: logger.info('Iters per test: {}'.format(len(test_loader))) # optimizer if args.optimizer == 'adam': optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) elif args.optimizer == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, momentum=0.9, nesterov=False) # restore parameters # import pdb; pdb.set_trace() if args.resume is not None: # import pdb; pdb.set_trace() print('resume from checkpoint') checkpt = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda(args.local_rank)) model.load_state_dict(checkpt["state_dict"]) if "optim_state_dict" in checkpt.keys(): optimizer.load_state_dict(checkpt["optim_state_dict"]) # Manually move optimizer state to device. for state in optimizer.state.values(): for k, v in state.items(): if torch.is_tensor(v): state[k] = cvt(v) # For visualization. if write_log: fixed_z = cvt(torch.randn(min(args.test_batch_size, 100), *data_shape)) if write_log: time_meter = utils.RunningAverageMeter(0.97) bpd_meter = utils.RunningAverageMeter(0.97) loss_meter = utils.RunningAverageMeter(0.97) steps_meter = utils.RunningAverageMeter(0.97) grad_meter = utils.RunningAverageMeter(0.97) tt_meter = utils.RunningAverageMeter(0.97) if not args.resume: best_loss = float("inf") itr = 0 wall_clock = 0. begin_epoch = 1 chkdir = args.save ''' elif args.resume and args.validate: chkdir = os.path.dirname(args.resume) wall_clock = 0 itr = 0 best_loss = 0.0 begin_epoch = 0 ''' else: chkdir = os.path.dirname(args.resume) filename = os.path.join(chkdir, 'test.csv') print(filename) tedf = pd.read_csv(os.path.join(chkdir, 'test.csv')) trdf = pd.read_csv(os.path.join(chkdir, 'training.csv')) # import pdb; pdb.set_trace() wall_clock = trdf['wall'].to_numpy()[-1] itr = trdf['itr'].to_numpy()[-1] best_loss = tedf['bpd'].min() begin_epoch = int(tedf['epoch'].to_numpy()[-1] + 1) # not exactly correct if args.distributed: if write_log: logger.info('Syncing machines before training') dist_utils.sum_tensor(torch.tensor([1.0]).float().cuda()) for epoch in range(begin_epoch, begin_epoch + 1): # compute test loss print('Evaluating') model.eval() if args.local_rank == 0: utils.makedirs(args.save) # import pdb; pdb.set_trace() if hasattr(model, 'module'): _state = model.module.state_dict() else: _state = model.state_dict() torch.save({ "args": args, "state_dict": _state, # model.module.state_dict() if torch.cuda.is_available() else model.state_dict(), "optim_state_dict": optimizer.state_dict(), "fixed_z": fixed_z.cpu() }, os.path.join(args.save, "checkpt_%d.pth" % epoch)) # save real and generate with different temperatures fig_num = 64 if True: # args.save_real: for i, (x, y) in enumerate(test_loader): if i < 100: pass elif i == 100: real = x.size(0) else: break if x.shape[0] > fig_num: x = x[:fig_num, ...] # import pdb; pdb.set_trace() fig_filename = os.path.join(chkdir, "real.jpg") save_image(x.float() / 255.0, fig_filename, nrow=8) if True: # args.generate: print('\nGenerating images... ') fixed_z = cvt(torch.randn(fig_num, *data_shape)) nb = int(np.ceil(np.sqrt(float(fixed_z.size(0))))) for t in [ 1.0, 0.99, 0.98, 0.97,0.96,0.95,0.93,0.92,0.90,0.85,0.8,0.75,0.7,0.65,0.6]: # visualize samples and density fig_filename = os.path.join(chkdir, "generated-T%g.jpg" % t) utils.makedirs(os.path.dirname(fig_filename)) generated_samples = model(t * fixed_z, reverse=True) x = unshift(generated_samples[0].view(-1, *data_shape), 8) save_image(x, fig_filename, nrow=nb)
default='./experiments/model-cifar-Resnet18', help='directory of model for saving checkpoint') parser.add_argument('--save-freq', '-s', default=50, type=int, metavar='N', help='save frequency') parser.add_argument('--coeff', type=float, default=0.99) args = parser.parse_args() # settings model_dir = args.model_dir utils.makedirs(model_dir) logger = utils.get_logger(logpath=os.path.join(model_dir, 'logs'), filepath=os.path.abspath(__file__)) logger.info(args) use_cuda = not args.no_cuda and torch.cuda.is_available() if use_cuda: torch.backends.cudnn.benchmark = True np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") kwargs = {'num_workers': 4, 'pin_memory': True} if use_cuda else {} ACTIVATION_FNS = { 'identity': base_layers.Identity, 'relu': torch.nn.ReLU, 'tanh': torch.nn.Tanh,
def __init__(self, save_adj_name, temperature, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._model_kwargs = kwargs.get('model') self._train_kwargs = kwargs.get('train') self.temperature = float(temperature) self.opt = self._train_kwargs.get('optimizer') self.max_grad_norm = self._train_kwargs.get('max_grad_norm', 1.) self.ANNEAL_RATE = 0.00003 self.temp_min = 0.1 self.save_adj_name = save_adj_name self.epoch_use_regularization = self._train_kwargs.get('epoch_use_regularization') self.num_sample = self._train_kwargs.get('num_sample') # logging. self._log_dir = self._get_log_dir(kwargs) self._writer = SummaryWriter('runs/' + self._log_dir) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) # data set self._data = utils.load_dataset(**self._data_kwargs) self.standard_scaler = self._data['scaler'] ### Feas if self._data_kwargs['dataset_dir'] == 'data/METR-LA': df = pd.read_hdf('./data/metr-la.h5') elif self._data_kwargs['dataset_dir'] == 'data/PEMS-BAY': df = pd.read_hdf('./data/pems-bay.h5') else: df = pd.read_csv('./data/pmu_normalized.csv', header=None) df = df.transpose() num_samples = df.shape[0] num_train = round(num_samples * 0.7) df = df[:num_train].values scaler = utils.StandardScaler(mean=df.mean(), std=df.std()) train_feas = scaler.transform(df) self._train_feas = torch.Tensor(train_feas).to(device) print(self._train_feas.shape) k = self._train_kwargs.get('knn_k') knn_metric = 'cosine' from sklearn.neighbors import kneighbors_graph g = kneighbors_graph(train_feas.T, k, metric=knn_metric) g = np.array(g.todense(), dtype=np.float32) self.adj_mx = torch.Tensor(g).to(device) self.num_nodes = int(self._model_kwargs.get('num_nodes', 1)) self.input_dim = int(self._model_kwargs.get('input_dim', 1)) self.seq_len = int(self._model_kwargs.get('seq_len')) # for the encoder self.output_dim = int(self._model_kwargs.get('output_dim', 1)) self.use_curriculum_learning = bool( self._model_kwargs.get('use_curriculum_learning', False)) self.horizon = int(self._model_kwargs.get('horizon', 1)) # for the decoder # setup model GTS_model = GTSModel(self.temperature, self._logger, **self._model_kwargs) self.GTS_model = GTS_model.cuda() if torch.cuda.is_available() else GTS_model self._logger.info("Model created") self._epoch_num = self._train_kwargs.get('epoch', 0) if self._epoch_num > 0: self.load_model()
parser.add_argument('--val-freq', type=int, default=500) parser.add_argument('--brute-val', action='store_true', default=False) parser.add_argument('--log-freq', type=int, default=100) parser.add_argument('--train-est-freq', type=int, default=None) args = parser.parse_args() args.val_batch_size = args.val_batch_size if args.val_batch_size else args.batch_size args.test_batch_size = args.test_batch_size if args.test_batch_size else args.batch_size args.train_est_freq = args.train_est_freq if args.train_est_freq else args.log_freq args.data_root = args.data_root if args.data_root else datasets.root save_path = os.path.join(args.save, 'checkpt.pth') log_path = os.path.join(args.save, 'logs') # Logger utils.makedirs(args.save) logger = utils.get_logger(logpath=log_path) logger.info(args) if args.fp64: torch.set_default_dtype(torch.float64) # noinspection PyPep8Naming def batch_iter(X, batch_size=args.batch_size, shuffle=False): """ X: feature tensor (shape: num_instances x num_features) """ if shuffle: idxs = torch.randperm(X.shape[0]) else: idxs = torch.arange(X.shape[0])
def __init__(self, is_training=True, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._train_kwargs = kwargs.get('train') self._test_kwargs = kwargs.get('test') self._model_kwargs = kwargs.get('model') self._alg_name = self._kwargs.get('alg') # data args self._raw_dataset_dir = self._data_kwargs.get('raw_dataset_dir') self._test_size = self._data_kwargs.get('test_size') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._logger.info(kwargs) # Model's Args self._model_type = self._model_kwargs.get('model_type') self._verified_percentage = self._model_kwargs.get( 'verified_percentage') self._rnn_units = self._model_kwargs.get('rnn_units') self._seq_len = self._model_kwargs.get('seq_len') self._horizon = self._model_kwargs.get('horizon') self._input_dim = self._model_kwargs.get('input_dim') self._input_shape = (self._seq_len, self._input_dim) self._output_dim = self._model_kwargs.get('output_dim') self._nodes = self._model_kwargs.get('num_nodes') self._num_rnn_layers = self._model_kwargs.get('num_rnn_layers') # Train's args self._drop_out = self._train_kwargs.get('dropout') self._epochs = self._train_kwargs.get('epochs') self._batch_size = self._data_kwargs.get('batch_size') self._optimizer = self._train_kwargs.get('optimizer') # Test's args self._run_times = self._test_kwargs.get('run_times') # Load data if self._model_type == 'ed' or self._model_type == 'encoder_decoder': self._data = utils.load_dataset_lstm_ed( seq_len=self._seq_len, horizon=self._horizon, input_dim=self._input_dim, output_dim=self._output_dim, raw_dataset_dir=self._raw_dataset_dir, r=self._verified_percentage, p=self._test_size, **kwargs) else: raise RuntimeError("Model must be lstm or encoder_decoder") self.callbacks_list = [] self._checkpoints = ModelCheckpoint(self._log_dir + "best_model.hdf5", monitor='val_loss', verbose=1, save_best_only=True, mode='auto', period=1) self.callbacks_list = [self._checkpoints] self._earlystop = EarlyStopping( monitor='val_loss', patience=self._train_kwargs.get('patience'), verbose=1, mode='auto') self.callbacks_list.append(self._earlystop) self._time_callback = TimeHistory() self.callbacks_list.append(self._time_callback) self.model = self._model_construction(is_training=is_training)
for i, horizon in enumerate(n_forwards): rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) mape = masked_mape_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) mae = masked_mae_np(preds=y_predicts[i].as_matrix(), labels=y_test.as_matrix(), null_val=0) line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae) logger.info(line) def main(args): traffic_reading_df = pd.read_hdf(args.traffic_reading_filename) eval_static(traffic_reading_df) eval_historical_average(traffic_reading_df, period=3) eval_var(traffic_reading_df, n_lags=3) if __name__ == '__main__': logger = utils.get_logger('data/model', 'Baseline') parser = argparse.ArgumentParser() parser.add_argument('--traffic_reading_filename', default="data/metr-la.h5", type=str, help='Path to the traffic Dataframe.') args = parser.parse_args() main(args)
parser.add_argument('--nworkers', type=int, default=2) #parser.add_argument('--nworkers', type=int, default=4) #parser.add_argument('--nworkers', type=int, default=4) parser.add_argument('--print-freq', help='Print progress every so iterations', type=int, default=20) parser.add_argument('--vis-freq', help='Visualize progress every so iterations', type=int, default=500) args = parser.parse_args() # Random seed if args.seed is None: args.seed = np.random.randint(100000) # logger utils.makedirs(args.save) logger = utils.get_logger(logpath=os.path.join(args.save, 'logs'), filepath=os.path.abspath(__file__)) logger.info(args) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') if device.type == 'cuda': logger.info('Found {} CUDA devices.'.format(torch.cuda.device_count())) for i in range(torch.cuda.device_count()): props = torch.cuda.get_device_properties(i) logger.info('{} \t Memory: {:.2f}GB'.format(props.name, props.total_memory / (1024**3))) else: logger.info('WARNING: Using device {}'.format(device)) np.random.seed(args.seed) torch.manual_seed(args.seed) if device.type == 'cuda':
import numpy as np import torch from bm_sequential import get_test_dataset as get_dataset from ctfp_tools import build_augmented_model_tabular from ctfp_tools import parse_arguments from ctfp_tools import run_ctfp_model as run_model from train_misc import ( create_regularization_fns, ) from train_misc import set_cnf_options torch.backends.cudnn.benchmark = True if __name__ == "__main__": args = parse_arguments() logger = utils.get_logger(logpath=os.path.join(args.save, "logs_test"), filepath=os.path.abspath(__file__)) if args.layer_type == "blend": logger.info( "!! Setting time_length from None to 1.0 due to use of Blend layers." ) args.time_length = 1.0 logger.info(args) # get deivce device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if args.use_cpu: device = torch.device("cpu") cvt = lambda x: x.type(torch.float32).to(device, non_blocking=True) # load dataset