def __init__(self, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._alg_name = kwargs.get('alg')
# data args
self._raw_dataset_dir = self._data_kwargs.get('raw_dataset_dir')
self._test_size = self._data_kwargs.get('test_size')
# logging
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
'info.log',
level=log_level)
self._logger.info(kwargs)
# Model's Args
self._verified_percentage = self._model_kwargs.get(
'verified_percentage')
self._seq_len = self._model_kwargs.get('seq_len')
self._horizon = self._model_kwargs.get('horizon')
self._nodes = self._model_kwargs.get('num_nodes')
# Test's args
self._run_times = self._test_kwargs.get('run_times')
# Load data
self._data = np.load(self._raw_dataset_dir)['data']
def __init__(self, obu_idx, total_obu_num, GLOBAL_ID_OBU_i_PK_i_map):
super().__init__()
self.ID_OBU_i = get_ID_obu_i(obu_idx)
self.total_num = total_obu_num
self.idx = str(obu_idx)
self.logger = get_logger("OBU_" + self.idx)
# ========= 生成属性 =========
self.r_i = random.randint(1, self.P)
self.x_i, self.y_i, self.PK_i = get_x_i_y_i_PK_i(
self.P, self.ID_OBU_i, self.s)
GLOBAL_ID_OBU_i_PK_i_map[self.ID_OBU_i] = self.PK_i
self.GLOBAL_ID_OBU_i_PK_i_map = GLOBAL_ID_OBU_i_PK_i_map
self.R_i = self.r_i * self.P
self.Ri_map = dict()
self.R = 0
self.logger.debug("x_i: {}, y_i: {}, PK_i: {}, R_i: {}".format(
self.x_i, self.y_i, self.PK_i, self.R_i))
# ========= 信号量定义 =======
self.sem_r_receive = Semaphore(0)
self.sem_rsp_msg = Semaphore(0)
# ========= 网络设置 ========
self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# ========= 回包 =========
self.rsp_msg = None
def __init__(self, **kwargs):
self._kwargs = kwargs
self._alg = kwargs.get('alg')
self._data_kwargs = kwargs.get('data')
self._train_kwargs = kwargs.get('train')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._base_dir = kwargs.get('base_dir')
self._epochs = self._train_kwargs.get('epochs')
# logging.
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level)
self._writer = tf.summary.FileWriter(self._log_dir)
self._logger.info(kwargs)
self._mon_ratio = float(self._kwargs.get('mon_ratio'))
# Model's args
self._seq_len = int(self._model_kwargs.get('seq_len'))
self._horizon = int(self._model_kwargs.get('horizon'))
self._input_dim = int(self._model_kwargs.get('input_dim'))
self._nodes = int(self._model_kwargs.get('num_nodes'))
self._rnn_units = int(self._model_kwargs.get('rnn_units'))
self._drop_out = float(self._train_kwargs.get('dropout'))
# Test's args
self._flow_selection = self._test_kwargs.get('flow_selection')
self._run_times = self._test_kwargs.get('run_times')
# Data preparation
self._day_size = self._data_kwargs.get('day_size')
def __init__(self, adj_mx, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._model_kwargs = kwargs.get('model')
self._train_kwargs = kwargs.get('train')
self.max_grad_norm = self._train_kwargs.get('max_grad_norm', 1.)
# logging.
self._log_dir = self._get_log_dir(kwargs)
self._writer = SummaryWriter(self._log_dir + '/runs/')
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level)
# data set
self._data = utils.load_dataset(**self._data_kwargs)
self.standard_scaler = self._data['scaler']
self.num_nodes = int(self._model_kwargs.get('num_nodes', 1))
self.input_dim = int(self._model_kwargs.get('input_dim', 1))
self.seq_len = int(self._model_kwargs.get('seq_len')) # for the encoder
self.output_dim = int(self._model_kwargs.get('output_dim', 1))
self.use_curriculum_learning = bool(
self._model_kwargs.get('use_curriculum_learning', False))
self.horizon = int(self._model_kwargs.get('horizon', 1)) # for the decoder
# setup model
garnn_model = GARNNModel(adj_mx, self._logger, **self._model_kwargs)
self.garnn_model = garnn_model.cuda() if torch.cuda.is_available() else garnn_model
self._logger.info("Model created")
self._epoch_num = self._train_kwargs.get('epoch', 0)
if self._epoch_num > 0:
self.load_model()
def __init__(self, GLOBAL_ID_OBU_i_PK_i_map, prime_nums):
super().__init__()
self.logger = get_logger("AMF")
self.prime_nums = prime_nums
# ========= 生成属性 =========
self.delta_i_list = list()
self.R = 0
self.R_AMF = 0
self.X_AMF = 0
self.W = 0
self.ID_OBU_i_V_i_map = dict()
self.GLOBAL_ID_OBU_i_PK_i_map = GLOBAL_ID_OBU_i_PK_i_map
self.R_i_msg_map = dict()
self.r_AMF = 0
self.GK = 0
self.x_amf, self.y_amf, self.PK_AMF = get_x_i_y_i_PK_i(
self.P, self.ID_AMF, self.s)
# ========= 信号量定义 =======
self.sem_qm_msg_receive = Semaphore(0)
# ========= 网络设置 ========
self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
def create_logger(self):
if self.logger or self.writer:
return
if not os.path.exists(self.log_root):
os.makedirs(self.log_root)
self.writer = SummaryWriter(log_dir=self.log_root)
opt_str = option_to_string(self.opt)
with open(os.path.join(self.log_root, 'config.txt'), 'w') as f:
f.writelines(opt_str)
self.logger = get_logger(self.log_root)
def train_toy(toy, load=True, nb_steps=20, nb_flow=1, folder=""):
device = "cpu"
logger = utils.get_logger(logpath=os.path.join(folder, toy, 'logs'),
filepath=os.path.abspath(__file__))
logger.info("Creating model...")
model = UMNNMAFFlow(nb_flow=nb_flow,
nb_in=2,
hidden_derivative=[50, 50, 50, 50],
hidden_embedding=[50, 50, 50, 50],
embedding_s=10,
nb_steps=nb_steps,
device=device).to(device)
logger.info("Model created.")
opt = torch.optim.Adam(model.parameters(), 1e-3, weight_decay=1e-5)
if load:
logger.info("Loading model...")
model.load_state_dict(torch.load(folder + toy + '/model.pt'))
model.train()
opt.load_state_dict(torch.load(folder + toy + '/ADAM.pt'))
logger.info("Model loaded.")
nb_samp = 1000
batch_size = 100
x_test = torch.tensor(toy_data.inf_train_gen(toy,
batch_size=1000)).to(device)
x = torch.tensor(toy_data.inf_train_gen(toy, batch_size=1000)).to(device)
for epoch in range(10000):
ll_tot = 0
start = timer()
for j in range(0, nb_samp, batch_size):
cur_x = torch.tensor(
toy_data.inf_train_gen(toy, batch_size=batch_size)).to(device)
ll, z = model.compute_ll(cur_x)
ll = -ll.mean()
ll_tot += ll.detach() / (nb_samp / batch_size)
loss = ll
opt.zero_grad()
loss.backward()
opt.step()
end = timer()
ll_test, _ = model.compute_ll(x_test)
ll_test = -ll_test.mean()
logger.info(
"epoch: {:d} - Train loss: {:4f} - Test loss: {:4f} - Elapsed time per epoch {:4f} (seconds)"
.format(epoch, ll_tot.item(), ll_test.item(), end - start))
if (epoch % 100) == 0:
summary_plots(x, x_test, folder, epoch, model, ll_tot, ll_test)
torch.save(model.state_dict(), folder + toy + '/model.pt')
torch.save(opt.state_dict(), folder + toy + '/ADAM.pt')
def __init__(self, catalog, override=False, schema='public'):
self.conn = SQLConnection('xmm', schema)
self.conn.autocommit = True
self.working_format = 'votable'
self.logger = get_logger('ICFx', log_filename='/tmp/icfx.log')
self.logger.setLevel(logging.DEBUG)
self.set_catalog(catalog)
self.override = override
self.source_catalogs = self.conn.execute_set("""
select distinct source_catalog from fields;""", quiet=False)
self.source_catalogs = [x[0] for x in self.source_catalogs]
sqllist.load_defaults()
sqllist.GLOBALS['c'] = self.catalog.CATALOG
def __init__(self, data_type, LOAD_INITIAL, adj_mx, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._model_kwargs = kwargs.get('model')
self._train_kwargs = kwargs.get('train')
self.max_grad_norm = self._train_kwargs.get('max_grad_norm', 1.)
# logging.
self._log_dir = self._get_log_dir(kwargs)
# self._writer = SummaryWriter('runs/' + self._log_dir)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
'info.log',
level=log_level)
# data set
self._data = utils.load_dataset(**self._data_kwargs)
self.standard_scaler = self._data['scaler']
self.num_nodes = int(self._model_kwargs.get('num_nodes', 1))
self.input_dim = int(self._model_kwargs.get('input_dim', 1))
self.seq_len = int(
self._model_kwargs.get('seq_len')) # for the encoder
self.output_dim = int(self._model_kwargs.get('output_dim', 1))
self.use_curriculum_learning = bool(
self._model_kwargs.get('use_curriculum_learning', False))
self.horizon = int(self._model_kwargs.get('horizon',
1)) # for the decoder
# setup model
# dcrnn_model = DCRNNModel(adj_mx, self._logger, **self._model_kwargs)
dcrnn_model = STGCN(self.num_nodes, self.input_dim, self.seq_len,
self.horizon)
self.dcrnn_model = dcrnn_model.cuda() if torch.cuda.is_available(
) else dcrnn_model
self._logger.info("Model created")
self._epoch_num = self._train_kwargs.get('epoch', 0)
# if self._epoch_num > 0: #事实上self._epoch_num的预设值确实为0
# self.load_model()
self.data_type = data_type
self.LOAD_INITIAL = LOAD_INITIAL
if LOAD_INITIAL:
self.load_lfx()
self.A_wave = torch.from_numpy(get_normalized_adj(adj_mx)).to(device)
def __init__(self, catalog, override=False, schema='public'):
self.conn = SQLConnection('xmm', schema)
self.conn.autocommit = True
self.working_format = 'votable'
self.logger = get_logger('ICFx', log_filename='/tmp/icfx.log')
self.logger.setLevel(logging.DEBUG)
self.set_catalog(catalog)
self.override = override
self.source_catalogs = self.conn.execute_set("""
select distinct source_catalog from fields;""",
quiet=False)
self.source_catalogs = [x[0] for x in self.source_catalogs]
sqllist.load_defaults()
sqllist.GLOBALS['c'] = self.catalog.CATALOG
def __init__(self, total_obu_num):
super().__init__()
self.ID_GL = get_ID_gl(time.time())
self.total_num = total_obu_num
self.logger = get_logger("GL")
# ========= 生成属性 =========
self.delta_i_list = list()
self.R = 0
self.W = 0
# ========= 信号量定义 =======
self.sem_delta_i_receive = Semaphore(0)
self.sem_msg_rsp_receive = Semaphore(0)
# ========= 网络设置 ========
self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
self.send_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# ========= 回包 =========
self.rsp_msg = None
viz = Visualizations(device)
##################################################################
#Load checkpoint and evaluate the model
if args.load is not None:
utils.get_ckpt_model(ckpt_path, model, device)
exit()
##################################################################
# Training
log_path = "logs/" + file_name + "_" + str(experimentID) + ".log"
if not os.path.exists("logs/"):
utils.makedirs("logs/")
logger = utils.get_logger(logpath=log_path,
filepath=os.path.abspath(__file__))
logger.info(input_command)
optimizer = optim.Adamax(model.parameters(), lr=args.lr)
num_batches = data_obj["n_train_batches"]
for itr in range(1, num_batches * (args.niters + 1)):
optimizer.zero_grad()
utils.update_learning_rate(optimizer,
decay_rate=0.999,
lowest=args.lr / 10)
wait_until_kl_inc = 10
if itr // num_batches < wait_until_kl_inc:
kl_coef = 0.
def __init__(self, is_training=False, **kwargs):
self._kwargs = kwargs
self._alg = kwargs.get('alg')
self._data_kwargs = kwargs.get('data')
self._train_kwargs = kwargs.get('train')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._base_dir = kwargs.get('base_dir')
self._epochs = self._train_kwargs.get('epochs')
# logging.
self._log_dir = get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
'info.log',
level=log_level)
self._writer = tf.summary.FileWriter(self._log_dir)
self._logger.info(kwargs)
self._mon_ratio = float(self._kwargs.get('mon_ratio'))
# Model's args
self._input_dim = int(self._model_kwargs.get('input_dim'))
self._nodes = int(self._model_kwargs.get('num_nodes'))
self._drop_out = float(self._train_kwargs.get('dropout'))
self.batch_size = int(self._data_kwargs['batch_size'])
# Test's args
self._flow_selection = self._test_kwargs.get('flow_selection')
self._run_times = self._test_kwargs.get('run_times')
# Data preparation
self._day_size = self._data_kwargs.get('day_size')
self._data = utils.load_dataset_gatlstm(
num_nodes=self._model_kwargs.get('num_nodes'),
input_dim=self._model_kwargs.get('input_dim'),
mon_ratio=self._mon_ratio,
scaler_type=self._kwargs.get('scaler'),
is_training=is_training,
**self._data_kwargs)
for k, v in self._data.items():
if hasattr(v, 'shape'):
self._logger.info((k, v.shape))
# Build models.
scaler = self._data['scaler']
if is_training:
self.model = GATLSTMModel(scaler=scaler,
batch_size=self.batch_size,
**self._model_kwargs)
else:
self.model = GATLSTMModel(scaler=scaler,
batch_size=1,
**self._model_kwargs)
# Learning rate.
max_to_keep = self._train_kwargs.get('max_to_keep', 100)
self.saver = tf.train.Saver(tf.global_variables(),
max_to_keep=max_to_keep)
self.summary_writer = tf.summary.FileWriter(self._log_dir)
# Log model statistics.
total_trainable_parameter = utils.get_total_trainable_parameter_size()
self._logger.info('Total number of trainable parameters: {:d}'.format(
total_trainable_parameter))
for var in tf.global_variables():
self._logger.debug('{}, {}'.format(var.name, var.get_shape()))
def __init__(self, args, adj_mx, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._model_kwargs = kwargs.get('model')
self._train_kwargs = kwargs.get('train')
# logging.
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
kwargs['name'] + '_info.log',
level=log_level)
self._writer = tf.summary.FileWriter(self._log_dir)
self._logger.info(kwargs)
# Data preparation
if self._data_kwargs.get('data_type') == 'npz':
self._data = utils.load_dataset(**self._data_kwargs)
elif self._data_kwargs.get('data_type') == 'csv':
self._data = utils.load_dataset_from_csv(**self._data_kwargs)
for k, v in self._data.items():
if hasattr(v, 'shape'):
self._logger.info((k, v.shape))
# Build models.
scaler = self._data['scaler']
with tf.name_scope('Train'):
with tf.variable_scope('DCRNN', reuse=False):
self._train_model = DCRNNModel(
args=args,
is_training=True,
scaler=scaler,
batch_size=self._data_kwargs['batch_size'],
adj_mx=adj_mx,
**self._model_kwargs)
with tf.name_scope('Test'):
with tf.variable_scope('DCRNN', reuse=True):
self._test_model = DCRNNModel(
args=args,
is_training=False,
scaler=scaler,
batch_size=self._data_kwargs['test_batch_size'],
adj_mx=adj_mx,
**self._model_kwargs)
# Learning rate.
self._lr = tf.get_variable('learning_rate',
shape=(),
initializer=tf.constant_initializer(0.01),
trainable=False)
self._new_lr = tf.placeholder(tf.float32,
shape=(),
name='new_learning_rate')
self._lr_update = tf.assign(self._lr, self._new_lr, name='lr_update')
# Configure optimizer
optimizer_name = self._train_kwargs.get('optimizer', 'adam').lower()
epsilon = float(self._train_kwargs.get('epsilon', 1e-3))
optimizer = tf.train.AdamOptimizer(self._lr, epsilon=epsilon)
if optimizer_name == 'sgd':
optimizer = tf.train.GradientDescentOptimizer(self._lr, )
elif optimizer_name == 'amsgrad':
optimizer = AMSGrad(self._lr, epsilon=epsilon)
# Calculate loss
output_dim = self._model_kwargs.get('output_dim')
preds = self._train_model.outputs
labels = self._train_model.labels[..., :output_dim]
null_val = 0.
self._loss_fn = masked_mae_loss(scaler, null_val)
self._mape_fn = masked_mape(scaler, null_val)
self._rmse_fn = masked_rmse_loss(scaler, null_val)
self._train_loss = self._loss_fn(preds=preds, labels=labels)
tvars = tf.trainable_variables()
grads = tf.gradients(self._train_loss, tvars)
max_grad_norm = kwargs['train'].get('max_grad_norm', 1.)
grads, _ = tf.clip_by_global_norm(grads, max_grad_norm)
global_step = tf.train.get_or_create_global_step()
self._train_op = optimizer.apply_gradients(zip(grads, tvars),
global_step=global_step,
name='train_op')
max_to_keep = self._train_kwargs.get('max_to_keep', 100)
self._epoch = 0
self._saver = tf.train.Saver(tf.global_variables(),
max_to_keep=max_to_keep)
# Log model statistics.
total_trainable_parameter = utils.get_total_trainable_parameter_size()
self._logger.info('Total number of trainable parameters: {:d}'.format(
total_trainable_parameter))
for var in tf.global_variables():
self._logger.debug('{}, {}'.format(var.name, var.get_shape()))
def __init__(self, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._train_kwargs = kwargs.get('train')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._alg_name = self._kwargs.get('alg')
# logging.
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
'info.log',
level=log_level)
self._logger.info(kwargs)
# Data's args
self._day_size = self._data_kwargs.get('day_size')
# Model's Args
self._model_type = self._model_kwargs.get('model_type')
self._rnn_units = self._model_kwargs.get('rnn_units')
self._seq_len = self._model_kwargs.get('seq_len')
self._horizon = self._model_kwargs.get('horizon')
self._input_dim = self._model_kwargs.get('input_dim')
self._input_shape = (self._seq_len, self._input_dim)
self._output_dim = self._model_kwargs.get('output_dim')
self._nodes = self._model_kwargs.get('num_nodes')
self._n_rnn_layers = self._model_kwargs.get('n_rnn_layers')
# Train's args
self._drop_out = self._train_kwargs.get('dropout')
self._epochs = self._train_kwargs.get('epochs')
self._batch_size = self._data_kwargs.get('batch_size')
# Test's args
self._run_times = self._test_kwargs.get('run_times')
self._flow_selection = self._test_kwargs.get('flow_selection')
self._test_size = self._test_kwargs.get('test_size')
self._results_path = self._test_kwargs.get('results_path')
self._mon_ratio = self._kwargs.get('mon_ratio')
# Load data
if self._model_type == 'lstm':
self._data = utils.load_dataset_lstm(seq_len=self._seq_len,
horizon=self._horizon,
input_dim=self._input_dim,
mon_ratio=self._mon_ratio,
test_size=self._test_size,
**self._data_kwargs)
elif self._model_type == 'ed' or self._model_type == 'encoder_decoder':
self._data = utils.load_dataset_lstm_ed(seq_len=self._seq_len,
horizon=self._horizon,
input_dim=self._input_dim,
mon_ratio=self._mon_ratio,
test_size=self._test_size,
**self._data_kwargs)
else:
raise RuntimeError("Model must be lstm or encoder_decoder")
for k, v in self._data.items():
if hasattr(v, 'shape'):
self._logger.info((k, v.shape))
# Model
self.callbacks_list = []
self._checkpoints = ModelCheckpoint(self._log_dir + "best_model.hdf5",
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto',
period=1)
self.callbacks_list = [self._checkpoints]
self._earlystop = EarlyStopping(
monitor='val_loss',
patience=self._train_kwargs.get('patience'),
verbose=1,
mode='auto')
self.callbacks_list.append(self._earlystop)
self._time_callback = TimeHistory()
self.callbacks_list.append(self._time_callback)
self.model = None
"""Various pre-processing steps that are applied before converting to a spacy doc."""
import shutil
from os.path import exists, join
from typing import Dict
import numpy as np
import pandas
import re
import wget
from abbreviations import schwartz_hearst
import constants
from lib.utils import get_logger
logger = get_logger("Preprocessing")
def load_unicode_mappings(datapath: str) -> Dict[int, str]:
"""
Load mappings between unicode and ascii characters.
datapath: path to data directory
"""
mappings = {}
filename = "entities.dat"
if not exists(join(datapath, filename)):
url = "https://structuredabstracts.nlm.nih.gov/Downloads/Structured-Abstracts-Labels-102615.txt"
logger.debug(
"File with unicode-to-ascii mappings not found. Downloading from %s.",
url)
filename = wget.download(url)
def __init__(self, is_training=True, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._train_kwargs = kwargs.get('train')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._alg_name = self._kwargs.get('alg')
# logging.
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level)
self._logger.info(kwargs)
self._mon_ratio = self._kwargs.get('mon_ratio')
# Data's args
self._day_size = self._data_kwargs.get('day_size')
# Model's Args
self._hidden = self._model_kwargs.get('rnn_units')
self._seq_len = self._model_kwargs.get('seq_len')
self._horizon = self._model_kwargs.get('horizon')
self._input_dim = self._model_kwargs.get('input_dim')
self._input_shape = (self._seq_len, self._input_dim)
self._output_dim = self._model_kwargs.get('output_dim')
self._nodes = self._model_kwargs.get('num_nodes')
self._r = self._model_kwargs.get('r')
# Train's args
self._drop_out = self._train_kwargs.get('dropout')
self._epochs = self._train_kwargs.get('epochs')
self._batch_size = self._data_kwargs.get('batch_size')
# Test's args
self._run_times = self._test_kwargs.get('run_times')
self._flow_selection = self._test_kwargs.get('flow_selection')
self._lamda = []
self._lamda.append(self._test_kwargs.get('lamda_0'))
self._lamda.append(self._test_kwargs.get('lamda_1'))
self._lamda.append(self._test_kwargs.get('lamda_2'))
# Load data
self._data = utils.load_dataset_fwbw_lstm_ed(seq_len=self._seq_len, horizon=self._horizon,
input_dim=self._input_dim,
mon_ratio=self._mon_ratio,
scaler_type=self._kwargs.get('scaler'),
**self._data_kwargs)
for k, v in self._data.items():
if hasattr(v, 'shape'):
self._logger.info((k, v.shape))
# Model
self.model = self.construct_fwbw_lstm_ed(is_training=is_training)
self.callbacks_list = []
self._checkpoints = ModelCheckpoint(
self._log_dir + "best_model.hdf5",
monitor='val_loss', verbose=1,
save_best_only=True,
mode='auto', period=1)
self.callbacks_list = [self._checkpoints]
self._earlystop = EarlyStopping(monitor='val_loss', patience=self._train_kwargs.get('patience'),
verbose=1, mode='auto')
self.callbacks_list.append(self._earlystop)
self._time_callback = TimeHistory()
self.callbacks_list.append(self._time_callback)
def __init__(self, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._train_kwargs = kwargs.get('train')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._alg_name = self._kwargs.get('alg')
# logging.
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
'info.log',
level=log_level)
self._logger.info(kwargs)
# Data's args
self._day_size = self._data_kwargs.get('day_size')
# Model's Args
self._seq_len = self._model_kwargs.get('seq_len')
self._horizon = self._model_kwargs.get('horizon')
self._output_dim = self._model_kwargs.get('output_dim')
self._nodes = self._model_kwargs.get('num_nodes')
self._wide = self._model_kwargs.get('wide')
self._high = self._model_kwargs.get('high')
self._channel = self._model_kwargs.get('channel')
self._filters = self._model_kwargs.get('filters')
self._kernel_size = self._model_kwargs.get('kernel_size')
self._strides = self._model_kwargs.get('strides')
self._input_shape = (self._seq_len, self._wide, self._high,
self._channel)
# Train's args
self._conv_dropout = self._train_kwargs.get('conv_dropout')
self._rnn_dropout = self._train_kwargs.get('rnn_dropout')
self._epochs = self._train_kwargs.get('epochs')
self._batch_size = self._data_kwargs.get('batch_size')
# Test's args
self._run_times = self._test_kwargs.get('run_times')
self._flow_selection = self._test_kwargs.get('flow_selection')
self._test_size = self._test_kwargs.get('test_size')
self._results_path = self._test_kwargs.get('results_path')
self._lamda = []
self._lamda.append(self._test_kwargs.get('lamda_0'))
self._lamda.append(self._test_kwargs.get('lamda_1'))
self._lamda.append(self._test_kwargs.get('lamda_2'))
self._mon_ratio = self._kwargs.get('mon_ratio')
# Load data
self._data = utils.load_dataset_conv_lstm(seq_len=self._seq_len,
wide=self._wide,
high=self._high,
channel=self._channel,
mon_ratio=self._mon_ratio,
test_size=self._test_size,
**self._data_kwargs)
for k, v in self._data.items():
if hasattr(v, 'shape'):
self._logger.info((k, v.shape))
# Model
self.model = None
self.callbacks_list = []
self._checkpoints = ModelCheckpoint(self._log_dir + "best_model.hdf5",
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto',
period=1)
self.callbacks_list = [self._checkpoints]
self._earlystop = EarlyStopping(
monitor='val_loss',
patience=self._train_kwargs.get('patience'),
verbose=1,
mode='auto')
self.callbacks_list.append(self._earlystop)
self._time_callback = TimeHistory()
self.callbacks_list.append(self._time_callback)
def train_toy(toy,
load=True,
nb_step_dual=300,
nb_steps=15,
folder="",
l1=1.,
nb_epoch=20000,
pre_heating_epochs=10,
nb_flow=3,
cond_type="Coupling",
emb_net=[150, 150, 150]):
logger = utils.get_logger(logpath=os.path.join(folder, toy, 'logs'),
filepath=os.path.abspath(__file__))
logger.info("Creating model...")
device = "cpu" if not (torch.cuda.is_available()) else "cuda:0"
nb_samp = 100
batch_size = 100
x_test = torch.tensor(toy_data.inf_train_gen(toy,
batch_size=1000)).to(device)
x = torch.tensor(toy_data.inf_train_gen(toy, batch_size=1000)).to(device)
dim = x.shape[1]
norm_type = "Affine"
save_name = norm_type + str(emb_net) + str(nb_flow)
solver = "CCParallel"
int_net = [150, 150, 150]
conditioner_type = cond_types[cond_type]
conditioner_args = {
"in_size": dim,
"hidden": emb_net[:-1],
"out_size": emb_net[-1]
}
if conditioner_type is DAGConditioner:
conditioner_args['l1'] = l1
conditioner_args['gumble_T'] = .5
conditioner_args['nb_epoch_update'] = nb_step_dual
conditioner_args["hot_encoding"] = True
normalizer_type = norm_types[norm_type]
if normalizer_type is MonotonicNormalizer:
normalizer_args = {
"integrand_net": int_net,
"cond_size": emb_net[-1],
"nb_steps": nb_steps,
"solver": solver
}
else:
normalizer_args = {}
model = buildFCNormalizingFlow(nb_flow, conditioner_type, conditioner_args,
normalizer_type, normalizer_args)
opt = torch.optim.Adam(model.parameters(), 1e-4, weight_decay=1e-5)
if load:
logger.info("Loading model...")
model.load_state_dict(
torch.load(folder + toy + '/' + save_name + 'model.pt'))
model.train()
opt.load_state_dict(
torch.load(folder + toy + '/' + save_name + 'ADAM.pt'))
logger.info("Model loaded.")
if True:
for step in model.steps:
step.conditioner.stoch_gate = True
step.conditioner.noise_gate = False
step.conditioner.gumble_T = .5
torch.autograd.set_detect_anomaly(True)
for epoch in range(nb_epoch):
loss_tot = 0
start = timer()
for j in range(0, nb_samp, batch_size):
cur_x = torch.tensor(
toy_data.inf_train_gen(toy, batch_size=batch_size)).to(device)
z, jac = model(cur_x)
loss = model.loss(z, jac)
loss_tot += loss.detach()
if math.isnan(loss.item()):
ll, z = model.compute_ll(cur_x)
print(ll)
print(z)
print(ll.max(), z.max())
exit()
opt.zero_grad()
loss.backward(retain_graph=True)
opt.step()
model.step(epoch, loss_tot)
end = timer()
z, jac = model(x_test)
ll = (model.z_log_density(z) + jac)
ll_test = -ll.mean()
dagness = max(model.DAGness())
logger.info(
"epoch: {:d} - Train loss: {:4f} - Test loss: {:4f} - <<DAGness>>: {:4f} - Elapsed time per epoch {:4f} (seconds)"
.format(epoch, loss_tot.item(), ll_test.item(), dagness,
end - start))
if epoch % 100 == 0 and False:
with torch.no_grad():
stoch_gate = model.getDag().stoch_gate
noise_gate = model.getDag().noise_gate
s_thresh = model.getDag().s_thresh
model.getDag().stoch_gate = False
model.getDag().noise_gate = False
model.getDag().s_thresh = True
for threshold in [.95, .1, .01, .0001, 1e-8]:
model.set_h_threshold(threshold)
# Valid loop
z, jac = model(x_test)
ll = (model.z_log_density(z) + jac)
ll_test = -ll.mean().item()
dagness = max(model.DAGness()).item()
logger.info(
"epoch: {:d} - Threshold: {:4f} - Valid log-likelihood: {:4f} - <<DAGness>>: {:4f}"
.format(epoch, threshold, ll_test, dagness))
model.getDag().stoch_gate = stoch_gate
model.getDag().noise_gate = noise_gate
model.getDag().s_thresh = s_thresh
model.set_h_threshold(0.)
if epoch % 500 == 0:
font = {'family': 'normal', 'weight': 'normal', 'size': 25}
matplotlib.rc('font', **font)
if toy in [
"2spirals-8gaussians", "4-2spirals-8gaussians",
"8-2spirals-8gaussians", "2gaussians", "4gaussians",
"2igaussians", "8gaussians"
] or True:
def compute_ll(x):
z, jac = model(x)
ll = (model.z_log_density(z) + jac)
return ll, z
with torch.no_grad():
npts = 100
plt.figure(figsize=(12, 12))
gs = gridspec.GridSpec(2,
2,
width_ratios=[3, 1],
height_ratios=[3, 1])
ax = plt.subplot(gs[0])
qz_1, qz_2 = vf.plt_flow(compute_ll,
ax,
npts=npts,
device=device)
plt.subplot(gs[1])
plt.plot(qz_1, np.linspace(-4, 4, npts))
plt.ylabel('$x_2$', fontsize=25, rotation=-90, labelpad=20)
plt.xticks([])
plt.subplot(gs[2])
plt.plot(np.linspace(-4, 4, npts), qz_2)
plt.xlabel('$x_1$', fontsize=25)
plt.yticks([])
plt.savefig("%s%s/flow_%s_%d.pdf" %
(folder, toy, save_name, epoch))
torch.save(model.state_dict(),
folder + toy + '/' + save_name + 'model.pt')
torch.save(opt.state_dict(),
folder + toy + '/' + save_name + 'ADAM.pt')
def run(args, kwargs):
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
args.model_signature = str(datetime.datetime.now())[0:19].replace(' ', '_')
args.model_signature = args.model_signature.replace(':', '_')
snapshots_path = os.path.join(args.out_dir, 'vae_' + args.dataset + '_')
snap_dir = snapshots_path + args.flow
if args.flow != 'no_flow':
snap_dir += '_' + 'num_flows_' + str(args.num_flows)
if args.flow == 'orthogonal':
snap_dir = snap_dir + '_num_vectors_' + str(args.num_ortho_vecs)
elif args.flow == 'orthogonalH':
snap_dir = snap_dir + '_num_householder_' + str(args.num_householder)
elif args.flow == 'iaf':
snap_dir = snap_dir + '_madehsize_' + str(args.made_h_size)
elif args.flow == 'permutation':
snap_dir = snap_dir + '_' + 'kernelsize_' + str(args.kernel_size)
elif args.flow == 'mixed':
snap_dir = snap_dir + '_' + 'num_householder_' + str(args.num_householder)
elif args.flow == 'cnf_rank':
snap_dir = snap_dir + '_rank_' + str(args.rank) + '_' + args.dims + '_num_blocks_' + str(args.num_blocks)
elif 'cnf' in args.flow:
snap_dir = snap_dir + '_' + args.dims + '_num_blocks_' + str(args.num_blocks)
if args.retrain_encoder:
snap_dir = snap_dir + '_retrain-encoder_'
elif args.evaluate:
snap_dir = snap_dir + '_evaluate_'
snap_dir = snap_dir + '__' + args.model_signature + '/'
args.snap_dir = snap_dir
if not os.path.exists(snap_dir):
os.makedirs(snap_dir)
# logger
utils.makedirs(args.snap_dir)
logger = utils.get_logger(logpath=os.path.join(args.snap_dir, 'logs'), filepath=os.path.abspath(__file__))
logger.info(args)
# SAVING
torch.save(args, snap_dir + args.flow + '.config')
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
if not args.evaluate:
# ==============================================================================================================
# SELECT MODEL
# ==============================================================================================================
# flow parameters and architecture choice are passed on to model through args
if args.flow == 'no_flow':
model = VAE.VAE(args)
elif args.flow == 'planar':
model = VAE.PlanarVAE(args)
elif args.flow == 'iaf':
model = VAE.IAFVAE(args)
elif args.flow == 'orthogonal':
model = VAE.OrthogonalSylvesterVAE(args)
elif args.flow == 'householder':
model = VAE.HouseholderSylvesterVAE(args)
elif args.flow == 'triangular':
model = VAE.TriangularSylvesterVAE(args)
elif args.flow == 'cnf':
model = CNFVAE.CNFVAE(args)
elif args.flow == 'cnf_bias':
model = CNFVAE.AmortizedBiasCNFVAE(args)
elif args.flow == 'cnf_hyper':
model = CNFVAE.HypernetCNFVAE(args)
elif args.flow == 'cnf_lyper':
model = CNFVAE.LypernetCNFVAE(args)
elif args.flow == 'cnf_rank':
model = CNFVAE.AmortizedLowRankCNFVAE(args)
else:
raise ValueError('Invalid flow choice')
if args.retrain_encoder:
logger.info(f"Initializing decoder from {args.model_path}")
dec_model = torch.load(args.model_path)
dec_sd = {}
for k, v in dec_model.state_dict().items():
if 'p_x' in k:
dec_sd[k] = v
model.load_state_dict(dec_sd, strict=False)
if args.cuda:
logger.info("Model on GPU")
model.cuda()
logger.info(model)
if args.retrain_encoder:
parameters = []
logger.info('Optimizing over:')
for name, param in model.named_parameters():
if 'p_x' not in name:
logger.info(name)
parameters.append(param)
else:
parameters = model.parameters()
optimizer = optim.Adamax(parameters, lr=args.learning_rate, eps=1.e-7)
# ==================================================================================================================
# TRAINING
# ==================================================================================================================
train_loss = []
val_loss = []
# for early stopping
best_loss = np.inf
best_bpd = np.inf
e = 0
epoch = 0
train_times = []
for epoch in range(1, args.epochs + 1):
t_start = time.time()
tr_loss = train(epoch, train_loader, model, optimizer, args, logger)
train_loss.append(tr_loss)
train_times.append(time.time() - t_start)
logger.info('One training epoch took %.2f seconds' % (time.time() - t_start))
v_loss, v_bpd = evaluate(val_loader, model, args, logger, epoch=epoch)
val_loss.append(v_loss)
# early-stopping
if v_loss < best_loss:
e = 0
best_loss = v_loss
if args.input_type != 'binary':
best_bpd = v_bpd
logger.info('->model saved<-')
torch.save(model, snap_dir + args.flow + '.model')
# torch.save(model, snap_dir + args.flow + '_' + args.architecture + '.model')
elif (args.early_stopping_epochs > 0) and (epoch >= args.warmup):
e += 1
if e > args.early_stopping_epochs:
break
if args.input_type == 'binary':
logger.info(
'--> Early stopping: {}/{} (BEST: loss {:.4f})\n'.format(e, args.early_stopping_epochs, best_loss)
)
else:
logger.info(
'--> Early stopping: {}/{} (BEST: loss {:.4f}, bpd {:.4f})\n'.
format(e, args.early_stopping_epochs, best_loss, best_bpd)
)
if math.isnan(v_loss):
raise ValueError('NaN encountered!')
train_loss = np.hstack(train_loss)
val_loss = np.array(val_loss)
plot_training_curve(train_loss, val_loss, fname=snap_dir + '/training_curve_%s.pdf' % args.flow)
# training time per epoch
train_times = np.array(train_times)
mean_train_time = np.mean(train_times)
std_train_time = np.std(train_times, ddof=1)
logger.info('Average train time per epoch: %.2f +/- %.2f' % (mean_train_time, std_train_time))
# ==================================================================================================================
# EVALUATION
# ==================================================================================================================
logger.info(args)
logger.info('Stopped after %d epochs' % epoch)
logger.info('Average train time per epoch: %.2f +/- %.2f' % (mean_train_time, std_train_time))
final_model = torch.load(snap_dir + args.flow + '.model')
validation_loss, validation_bpd = evaluate(val_loader, final_model, args, logger)
else:
validation_loss = "N/A"
validation_bpd = "N/A"
logger.info(f"Loading model from {args.model_path}")
final_model = torch.load(args.model_path)
test_loss, test_bpd = evaluate(test_loader, final_model, args, logger, testing=True)
logger.info('FINAL EVALUATION ON VALIDATION SET. ELBO (VAL): {:.4f}'.format(validation_loss))
logger.info('FINAL EVALUATION ON TEST SET. NLL (TEST): {:.4f}'.format(test_loss))
if args.input_type != 'binary':
logger.info('FINAL EVALUATION ON VALIDATION SET. ELBO (VAL) BPD : {:.4f}'.format(validation_bpd))
logger.info('FINAL EVALUATION ON TEST SET. NLL (TEST) BPD: {:.4f}'.format(test_bpd))
def train(dataset="MNIST",
load=True,
nb_step_dual=100,
nb_steps=20,
path="",
l1=.1,
nb_epoch=10000,
b_size=100,
int_net=[50, 50, 50],
all_args=None,
file_number=None,
train=True,
solver="CC",
weight_decay=1e-5,
learning_rate=1e-3,
batch_per_optim_step=1,
n_gpu=1,
norm_type='Affine',
nb_flow=[1],
hot_encoding=True,
prior_A_kernel=None,
conditioner="DAG",
emb_net=None):
logger = utils.get_logger(logpath=os.path.join(path, 'logs'),
filepath=os.path.abspath(__file__))
logger.info(str(all_args))
if load:
file_number = "_" + file_number if file_number is not None else ""
batch_size = b_size
best_valid_loss = np.inf
logger.info("Loading data...")
train_loader, valid_loader, test_loader = load_data(dataset, batch_size)
if len(dataset) == 6 and dataset[:5] == 'MNIST':
dataset = "MNIST"
alpha = 1e-6 if dataset == "MNIST" else .05
logger.info("Data loaded.")
master_device = "cuda:0" if torch.cuda.is_available() else "cpu"
# ----------------------- Model Definition ------------------- #
logger.info("Creating model...")
if norm_type == 'Affine':
normalizer_type = AffineNormalizer
normalizer_args = {}
else:
normalizer_type = MonotonicNormalizer
normalizer_args = {
"integrand_net": int_net,
"nb_steps": 15,
"solver": solver
}
if conditioner == "DAG":
conditioner_type = DAGConditioner
if dataset == "MNIST":
inner_model = buildMNISTNormalizingFlow(
nb_flow,
normalizer_type,
normalizer_args,
l1,
nb_epoch_update=nb_step_dual,
hot_encoding=hot_encoding,
prior_kernel=prior_A_kernel)
elif dataset == "CIFAR10":
inner_model = buildCIFAR10NormalizingFlow(
nb_flow,
normalizer_type,
normalizer_args,
l1,
nb_epoch_update=nb_step_dual,
hot_encoding=hot_encoding)
else:
logger.info("Wrong dataset name. Training aborted.")
exit()
else:
dim = 28**2 if dataset == "MNIST" else 32 * 32 * 3
conditioner_type = cond_types[conditioner]
conditioner_args = {
"in_size": dim,
"hidden": emb_net[:-1],
"out_size": emb_net[-1]
}
if norm_type == 'Monotonic':
normalizer_args["cond_size"] = emb_net[-1]
inner_model = buildFCNormalizingFlow(nb_flow[0], conditioner_type,
conditioner_args, normalizer_type,
normalizer_args)
model = nn.DataParallel(inner_model,
device_ids=list(range(n_gpu))).to(master_device)
logger.info(str(model))
pytorch_total_params = sum(p.numel() for p in model.parameters())
logger.info("Number of parameters: %d" % pytorch_total_params)
opt = torch.optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
if load:
logger.info("Loading model...")
model.load_state_dict(
torch.load(path + '/model%s.pt' % file_number,
map_location={"cuda:0": master_device}))
model.train()
opt.load_state_dict(
torch.load(path + '/ADAM%s.pt' % file_number,
map_location={"cuda:0": master_device}))
if master_device != "cpu":
for state in opt.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.cuda()
logger.info("...Model built.")
logger.info("Training starts:")
if load:
for conditioner in model.module.getConditioners():
conditioner.alpha = conditioner.getAlpha()
# ----------------------- Main Loop ------------------------- #
for epoch in range(nb_epoch):
ll_tot = 0
start = timer()
if train:
model.to(master_device)
# ----------------------- Training Loop ------------------------- #
for batch_idx, (cur_x, target) in enumerate(train_loader):
cur_x = cur_x.view(batch_size, -1).float().to(master_device)
for normalizer in model.module.getNormalizers():
if type(normalizer) is MonotonicNormalizer:
normalizer.nb_steps = nb_steps + torch.randint(
0, 10, [1])[0].item()
z, jac = model(cur_x)
loss = model.module.loss(z,
jac) / (batch_per_optim_step * n_gpu)
if math.isnan(loss.item()):
print("Error Nan in loss")
print("Dagness:", model.module.DAGness())
exit()
ll_tot += loss.detach()
if batch_idx % batch_per_optim_step == 0:
opt.zero_grad()
loss.backward(retain_graph=True)
if (batch_idx + 1) % batch_per_optim_step == 0:
opt.step()
with torch.no_grad():
print("Dagness:", model.module.DAGness())
ll_tot /= (batch_idx + 1)
torch.cuda.empty_cache()
model.module.step(epoch, ll_tot)
else:
ll_tot = 0.
# ----------------------- Valid Loop ------------------------- #
ll_test = 0.
bpp_test = 0.
model.to(master_device)
with torch.no_grad():
for normalizer in model.module.getNormalizers():
if type(normalizer) is MonotonicNormalizer:
normalizer.nb_steps = 150
for batch_idx, (cur_x, target) in enumerate(valid_loader):
cur_x = cur_x.view(batch_size, -1).float().to(master_device)
z, jac = model(cur_x)
ll = (model.module.z_log_density(z) + jac)
ll_test += ll.mean().item()
bpp_test += compute_bpp(
ll,
cur_x.view(batch_size, -1).float().to(master_device),
alpha).mean().item()
ll_test /= batch_idx + 1
bpp_test /= batch_idx + 1
end = timer()
dagness = max(model.module.DAGness())
logger.info(
"epoch: {:d} - Train loss: {:4f} - Valid log-likelihood: {:4f} - Valid BPP {:4f} - <<DAGness>>: {:4f} "
"- Elapsed time per epoch {:4f} (seconds)".format(
epoch, ll_tot, ll_test, bpp_test, dagness, end - start))
if model.module.isInvertible() and -ll_test < best_valid_loss:
logger.info("------- New best validation loss --------")
torch.save(model.state_dict(), path + '/best_model.pt')
best_valid_loss = -ll_test
# Valid loop
ll_test = 0.
for batch_idx, (cur_x, target) in enumerate(test_loader):
z, jac = model(
cur_x.view(batch_size, -1).float().to(master_device))
ll = (model.module.z_log_density(z) + jac)
ll_test += ll.mean().item()
bpp_test += compute_bpp(
ll,
cur_x.view(batch_size, -1).float().to(master_device),
alpha).mean().item()
ll_test /= batch_idx + 1
bpp_test /= batch_idx + 1
logger.info(
"epoch: {:d} - Test log-likelihood: {:4f} - Test BPP {:4f} - <<DAGness>>: {:4f}"
.format(epoch, ll_test, bpp_test, dagness))
if epoch % 10 == 0 and conditioner_type is DAGConditioner:
stoch_gate, noise_gate, s_thresh = [], [], []
for conditioner in model.module.getConditioners():
stoch_gate.append(conditioner.stoch_gate)
noise_gate.append(conditioner.noise_gate)
s_thresh.append(conditioner.s_thresh)
conditioner.stoch_gate = False
conditioner.noise_gate = False
conditioner.s_thresh = True
for threshold in [.95, .5, .1, .01, .0001]:
for conditioner in model.module.getConditioners():
conditioner.h_thresh = threshold
# Valid loop
ll_test = 0.
bpp_test = 0.
for batch_idx, (cur_x, target) in enumerate(valid_loader):
cur_x = cur_x.view(batch_size,
-1).float().to(master_device)
z, jac = model(cur_x)
ll = (model.module.z_log_density(z) + jac)
ll_test += ll.mean().item()
bpp_test += compute_bpp(
ll,
cur_x.view(batch_size,
-1).float().to(master_device),
alpha).mean().item()
ll_test /= batch_idx + 1
bpp_test /= batch_idx + 1
dagness = max(model.module.DAGness())
logger.info(
"epoch: {:d} - Threshold: {:4f} - Valid log-likelihood: {:4f} - Valid BPP {:4f} - <<DAGness>>: {:4f}"
.format(epoch, threshold, ll_test, bpp_test, dagness))
for i, conditioner in enumerate(
model.module.getConditioners()):
conditioner.h_thresh = 0.
conditioner.stoch_gate = stoch_gate[i]
conditioner.noise_gate = noise_gate[i]
conditioner.s_thresh = s_thresh[i]
in_s = 784 if dataset == "MNIST" else 3 * 32 * 32
a_tmp = model.module.getConditioners()[0].soft_thresholded_A()[
0, :]
a_tmp = a_tmp.view(
28,
28).cpu().numpy() if dataset == "MNIST" else a_tmp.view(
3, 32, 32).cpu().numpy()
fig, ax = plt.subplots()
mat = ax.matshow(a_tmp)
plt.colorbar(mat)
current_cmap = matplotlib.cm.get_cmap()
current_cmap.set_bad(color='red')
mat.set_clim(0, 1.)
def update(i):
A = model.module.getConditioners()[0].soft_thresholded_A()[
i, :].cpu().numpy()
A[i] = np.nan
if dataset == "MNIST":
A = A.reshape(28, 28)
elif dataset == "CIFAR10":
A = A.reshape(3, 32, 32)
mat.set_data(A)
return mat
# Set up formatting for the movie files
Writer = animation.writers['ffmpeg']
writer = Writer(fps=15,
metadata=dict(artist='Me'),
bitrate=1800)
ani = animation.FuncAnimation(fig,
update,
range(in_s),
interval=100,
save_count=0)
ani.save(path + '/A_epoch_%d.mp4' % epoch, writer=writer)
deg_out = (
model.module.getConditioners()[0].soft_thresholded_A() >
0.).sum(0).cpu().numpy()
deg_in = (
model.module.getConditioners()[0].soft_thresholded_A() >
0.).sum(1).cpu().numpy()
fig, ax = plt.subplots(1, 2, figsize=(12, 6))
if dataset == "MNIST":
shape = (28, 28)
elif dataset == "CIFAR10":
shape = (3, 32, 32)
res0 = ax[0].matshow(np.log(deg_in).reshape(shape))
ax[0].set(title="In degrees")
fig.colorbar(res0, ax=ax[0])
res1 = ax[1].matshow(np.log(deg_out.reshape(shape)))
ax[1].set(title="Out degrees")
fig.colorbar(res1, ax=ax[1])
plt.savefig(path + '/A_degrees_epoch_%d.png' % epoch)
if model.module.isInvertible():
with torch.no_grad():
n_images = 16
in_s = 28**2
for T in [.1, .25, .5, .75, 1.]:
z = torch.randn(n_images,
in_s).to(device=master_device) * T
x = model.module.invert(z)
print((z - model(x)[0]).abs().mean())
grid_img = torchvision.utils.make_grid(x.view(
n_images, 1, 28, 28),
nrow=4)
torchvision.utils.save_image(
grid_img, path + '/images_%d_%f.png' % (epoch, T))
if epoch % nb_step_dual == 0:
logger.info("Saving model N°%d" % epoch)
torch.save(model.state_dict(), path + '/model_%d.pt' % epoch)
torch.save(opt.state_dict(), path + '/ADAM_%d.pt' % epoch)
torch.save(model.state_dict(), path + '/model.pt')
torch.save(opt.state_dict(), path + '/ADAM.pt')
torch.cuda.empty_cache()
def train(dataset="POWER",
load=True,
nb_step_dual=100,
nb_steps=20,
path="",
l1=.1,
nb_epoch=10000,
int_net=[200, 200, 200],
emb_net=[200, 200, 200],
b_size=100,
all_args=None,
file_number=None,
train=True,
solver="CC",
nb_flow=1,
weight_decay=1e-5,
learning_rate=1e-3,
cond_type='DAG',
norm_type='affine'):
logger = utils.get_logger(logpath=os.path.join(path, 'logs'),
filepath=os.path.abspath(__file__))
logger.info(str(all_args))
logger.info("Creating model...")
device = "cpu" if not (torch.cuda.is_available()) else "cuda:0"
if load:
#train = False
file_number = "_" + file_number if file_number is not None else ""
batch_size = b_size
logger.info("Loading data...")
data = load_data(dataset)
data.trn.x = torch.from_numpy(data.trn.x).to(device)
data.val.x = torch.from_numpy(data.val.x).to(device)
data.tst.x = torch.from_numpy(data.tst.x).to(device)
logger.info("Data loaded.")
dim = data.trn.x.shape[1]
conditioner_type = cond_types[cond_type]
conditioner_args = {
"in_size": dim,
"hidden": emb_net[:-1],
"out_size": emb_net[-1]
}
if conditioner_type is DAGConditioner:
conditioner_args['l1'] = l1
conditioner_args['gumble_T'] = .5
conditioner_args['nb_epoch_update'] = nb_step_dual
conditioner_args["hot_encoding"] = True
normalizer_type = norm_types[norm_type]
if normalizer_type is MonotonicNormalizer:
normalizer_args = {
"integrand_net": int_net,
"cond_size": emb_net[-1],
"nb_steps": nb_steps,
"solver": solver
}
else:
normalizer_args = {}
model = buildFCNormalizingFlow(nb_flow, conditioner_type, conditioner_args,
normalizer_type, normalizer_args)
best_valid_loss = np.inf
opt = torch.optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
if load:
logger.info("Loading model...")
model.load_state_dict(
torch.load(path + '/model%s.pt' % file_number,
map_location={"cuda:0": device}))
model.train()
if os.path.isfile(path + '/ADAM%s.pt'):
opt.load_state_dict(
torch.load(path + '/ADAM%s.pt' % file_number,
map_location={"cuda:0": device}))
if device != "cpu":
for state in opt.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.cuda()
#x = data.trn.x[:20]
#print(x, model(x))
#exit()
for epoch in range(nb_epoch):
ll_tot = 0
start = timer()
# Update constraints
if conditioner_type is DAGConditioner:
with torch.no_grad():
for conditioner in model.getConditioners():
conditioner.constrainA(zero_threshold=0.)
# Training loop
model.to(device)
if train:
for i, cur_x in enumerate(
batch_iter(data.trn.x, shuffle=True,
batch_size=batch_size)):
if normalizer_type is MonotonicNormalizer:
for normalizer in model.getNormalizers():
normalizer.nb_steps = nb_steps + torch.randint(
0, 10, [1])[0].item()
z, jac = model(cur_x)
#print(z.mean(), jac.mean())
loss = model.loss(z, jac)
if math.isnan(loss.item()) or math.isinf(loss.abs().item()):
torch.save(model.state_dict(), path + '/NANmodel.pt')
print("Error NAN in loss")
exit()
ll_tot += loss.detach()
opt.zero_grad()
loss.backward(retain_graph=True)
opt.step()
ll_tot /= i + 1
model.step(epoch, ll_tot)
else:
ll_tot = 0.
# Valid loop
ll_test = 0.
with torch.no_grad():
if normalizer_type is MonotonicNormalizer:
for normalizer in model.getNormalizers():
normalizer.nb_steps = nb_steps + 20
for i, cur_x in enumerate(
batch_iter(data.val.x, shuffle=True,
batch_size=batch_size)):
z, jac = model(cur_x)
ll = (model.z_log_density(z) + jac)
ll_test += ll.mean().item()
ll_test /= i + 1
end = timer()
dagness = max(model.DAGness())
logger.info(
"epoch: {:d} - Train loss: {:4f} - Valid log-likelihood: {:4f} - <<DAGness>>: {:4f} - Elapsed time per epoch {:4f} (seconds)"
.format(epoch, ll_tot.item(), ll_test, dagness, end - start))
if dagness < 1e-20 and -ll_test < best_valid_loss:
logger.info("------- New best validation loss --------")
torch.save(model.state_dict(), path + '/best_model.pt')
best_valid_loss = -ll_test
# Valid loop
ll_test = 0.
for i, cur_x in enumerate(
batch_iter(data.tst.x,
shuffle=True,
batch_size=batch_size)):
z, jac = model(cur_x)
ll = (model.z_log_density(z) + jac)
ll_test += ll.mean().item()
ll_test /= i + 1
logger.info(
"epoch: {:d} - Test log-likelihood: {:4f} - <<DAGness>>: {:4f}"
.format(epoch, ll_test, dagness))
if epoch % 10 == 0 and conditioner_type is DAGConditioner:
stoch_gate, noise_gate, s_thresh = [], [], []
for conditioner in model.getConditioners():
stoch_gate.append(conditioner.stoch_gate)
noise_gate.append(conditioner.noise_gate)
s_thresh.append(conditioner.s_thresh)
conditioner.stoch_gate = False
conditioner.noise_gate = False
conditioner.s_thresh = True
for threshold in [.95, .5, .1, .01, .0001]:
for conditioner in model.getConditioners():
conditioner.h_thresh = threshold
# Valid loop
ll_test = 0.
for i, cur_x in enumerate(
batch_iter(data.val.x,
shuffle=True,
batch_size=batch_size)):
z, jac = model(cur_x)
ll = (model.z_log_density(z) + jac)
ll_test += ll.mean().item()
ll_test /= i
dagness = max(model.DAGness())
logger.info(
"epoch: {:d} - Threshold: {:4f} - Valid log-likelihood: {:4f} - <<DAGness>>: {:4f}"
.format(epoch, threshold, ll_test, dagness))
for i, conditioner in enumerate(model.getConditioners()):
conditioner.h_thresh = threshold
conditioner.stoch_gate = stoch_gate[i]
conditioner.noise_gate = noise_gate[i]
conditioner.s_thresh = s_thresh[i]
torch.save(model.state_dict(), path + '/model_%d.pt' % epoch)
torch.save(opt.state_dict(), path + '/ADAM_%d.pt' % epoch)
if dataset == "proteins" and conditioner_type is DAGConditioner:
torch.save(
model.getConditioners[0].soft_thresholded_A().detach().cpu(
), path + '/A_%d.pt' % epoch)
torch.save(model.state_dict(), path + '/model.pt')
torch.save(opt.state_dict(), path + '/ADAM.pt')
def main(args):
cfg = read_cfg_file(args.config_filename)
log_dir = _get_log_dir(cfg)
log_level = cfg.get('log_level', 'INFO')
logger = utils.get_logger(log_dir, __name__, 'info.log', level=log_level)
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# all edge_index in same dataset is same
# edge_index = adjacency_to_edge_index(adj_mx) # alreay added self-loop
logger.info(cfg)
# batch_size = cfg['data']['batch_size']
# test_batch_size = cfg['data']['test_batch_size']
# edge_index = utils.load_pickle(cfg['data']['edge_index_pkl_filename'])
hz = cfg['data'].get('name', 'nothz') == 'hz'
adj_mx_list = []
graph_pkl_filename = cfg['data']['graph_pkl_filename']
if not isinstance(graph_pkl_filename, list):
graph_pkl_filename = [graph_pkl_filename]
src = []
dst = []
for g in graph_pkl_filename:
if hz:
adj_mx = utils.load_graph_data_hz(g)
else:
_, _, adj_mx = utils.load_graph_data(g)
for i in range(len(adj_mx)):
adj_mx[i, i] = 0
adj_mx_list.append(adj_mx)
adj_mx = np.stack(adj_mx_list, axis=-1)
if cfg['model'].get('norm', False):
print('row normalization')
adj_mx = adj_mx / (adj_mx.sum(axis=0) + 1e-18)
src, dst = adj_mx.sum(axis=-1).nonzero()
edge_index = torch.tensor([src, dst], dtype=torch.long, device=device)
edge_attr = torch.tensor(adj_mx[adj_mx.sum(axis=-1) != 0],
dtype=torch.float,
device=device)
output_dim = cfg['model']['output_dim']
for i in range(adj_mx.shape[-1]):
logger.info(adj_mx[..., i])
# print(adj_mx.shape) (207, 207)
if hz:
dataset = utils.load_dataset_hz(**cfg['data'],
scaler_axis=(0, 1, 2, 3))
else:
dataset = utils.load_dataset(**cfg['data'])
for k, v in dataset.items():
if hasattr(v, 'shape'):
logger.info((k, v.shape))
model = Net(cfg).to(device)
model.load_state_dict(torch.load(cfg['model']['save_path']), strict=False)
evaluate(model=model,
dataset=dataset,
dataset_type='test',
edge_index=edge_index,
edge_attr=edge_attr,
device=device,
output_dim=output_dim,
logger=logger,
cfg=cfg,
format_result=True)
def main():
# os.system('shutdown -c') # cancel previous shutdown command
if write_log:
utils.makedirs(args.save)
logger = utils.get_logger(logpath=os.path.join(args.save, 'logs'), filepath=os.path.abspath(__file__))
logger.info(args)
args_file_path = os.path.join(args.save, 'args.yaml')
with open(args_file_path, 'w') as f:
yaml.dump(vars(args), f, default_flow_style=False)
if args.distributed:
if write_log: logger.info('Distributed initializing process group')
torch.cuda.set_device(args.local_rank)
distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
world_size=dist_utils.env_world_size(), rank=env_rank())
assert (dist_utils.env_world_size() == distributed.get_world_size())
if write_log: logger.info("Distributed: success (%d/%d)" % (args.local_rank, distributed.get_world_size()))
device = torch.device("cuda:%d" % torch.cuda.current_device() if torch.cuda.is_available() else "cpu")
else:
device = torch.cuda.current_device() #
# import pdb; pdb.set_trace()
cvt = lambda x: x.type(torch.float32).to(device, non_blocking=True)
# load dataset
train_loader, test_loader, data_shape = get_dataset(args)
trainlog = os.path.join(args.save, 'training.csv')
testlog = os.path.join(args.save, 'test.csv')
traincolumns = ['itr', 'wall', 'itr_time', 'loss', 'bpd', 'fe', 'total_time', 'grad_norm']
testcolumns = ['wall', 'epoch', 'eval_time', 'bpd', 'fe', 'total_time', 'transport_cost']
# build model
regularization_fns, regularization_coeffs = create_regularization_fns(args)
model = create_model(args, data_shape, regularization_fns).cuda()
if args.distributed: model = dist_utils.DDP(model,
device_ids=[args.local_rank],
output_device=args.local_rank)
traincolumns = append_regularization_keys_header(traincolumns, regularization_fns)
if not args.resume and write_log:
with open(trainlog, 'w') as f:
csvlogger = csv.DictWriter(f, traincolumns)
csvlogger.writeheader()
with open(testlog, 'w') as f:
csvlogger = csv.DictWriter(f, testcolumns)
csvlogger.writeheader()
set_cnf_options(args, model)
if write_log: logger.info(model)
if write_log: logger.info("Number of trainable parameters: {}".format(count_parameters(model)))
if write_log: logger.info('Iters per train epoch: {}'.format(len(train_loader)))
if write_log: logger.info('Iters per test: {}'.format(len(test_loader)))
# optimizer
if args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
elif args.optimizer == 'sgd':
optimizer = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.weight_decay, momentum=0.9,
nesterov=False)
# restore parameters
# import pdb; pdb.set_trace()
if args.resume is not None:
# import pdb; pdb.set_trace()
print('resume from checkpoint')
checkpt = torch.load(args.resume, map_location=lambda storage, loc: storage.cuda(args.local_rank))
model.load_state_dict(checkpt["state_dict"])
if "optim_state_dict" in checkpt.keys():
optimizer.load_state_dict(checkpt["optim_state_dict"])
# Manually move optimizer state to device.
for state in optimizer.state.values():
for k, v in state.items():
if torch.is_tensor(v):
state[k] = cvt(v)
# For visualization.
if write_log: fixed_z = cvt(torch.randn(min(args.test_batch_size, 100), *data_shape))
if write_log:
time_meter = utils.RunningAverageMeter(0.97)
bpd_meter = utils.RunningAverageMeter(0.97)
loss_meter = utils.RunningAverageMeter(0.97)
steps_meter = utils.RunningAverageMeter(0.97)
grad_meter = utils.RunningAverageMeter(0.97)
tt_meter = utils.RunningAverageMeter(0.97)
if not args.resume:
best_loss = float("inf")
itr = 0
wall_clock = 0.
begin_epoch = 1
chkdir = args.save
'''
elif args.resume and args.validate:
chkdir = os.path.dirname(args.resume)
wall_clock = 0
itr = 0
best_loss = 0.0
begin_epoch = 0
'''
else:
chkdir = os.path.dirname(args.resume)
filename = os.path.join(chkdir, 'test.csv')
print(filename)
tedf = pd.read_csv(os.path.join(chkdir, 'test.csv'))
trdf = pd.read_csv(os.path.join(chkdir, 'training.csv'))
# import pdb; pdb.set_trace()
wall_clock = trdf['wall'].to_numpy()[-1]
itr = trdf['itr'].to_numpy()[-1]
best_loss = tedf['bpd'].min()
begin_epoch = int(tedf['epoch'].to_numpy()[-1] + 1) # not exactly correct
if args.distributed:
if write_log: logger.info('Syncing machines before training')
dist_utils.sum_tensor(torch.tensor([1.0]).float().cuda())
for epoch in range(begin_epoch, begin_epoch + 1):
# compute test loss
print('Evaluating')
model.eval()
if args.local_rank == 0:
utils.makedirs(args.save)
# import pdb; pdb.set_trace()
if hasattr(model, 'module'):
_state = model.module.state_dict()
else:
_state = model.state_dict()
torch.save({
"args": args,
"state_dict": _state, # model.module.state_dict() if torch.cuda.is_available() else model.state_dict(),
"optim_state_dict": optimizer.state_dict(),
"fixed_z": fixed_z.cpu()
}, os.path.join(args.save, "checkpt_%d.pth" % epoch))
# save real and generate with different temperatures
fig_num = 64
if True: # args.save_real:
for i, (x, y) in enumerate(test_loader):
if i < 100:
pass
elif i == 100:
real = x.size(0)
else:
break
if x.shape[0] > fig_num:
x = x[:fig_num, ...]
# import pdb; pdb.set_trace()
fig_filename = os.path.join(chkdir, "real.jpg")
save_image(x.float() / 255.0, fig_filename, nrow=8)
if True: # args.generate:
print('\nGenerating images... ')
fixed_z = cvt(torch.randn(fig_num, *data_shape))
nb = int(np.ceil(np.sqrt(float(fixed_z.size(0)))))
for t in [ 1.0, 0.99, 0.98, 0.97,0.96,0.95,0.93,0.92,0.90,0.85,0.8,0.75,0.7,0.65,0.6]:
# visualize samples and density
fig_filename = os.path.join(chkdir, "generated-T%g.jpg" % t)
utils.makedirs(os.path.dirname(fig_filename))
generated_samples = model(t * fixed_z, reverse=True)
x = unshift(generated_samples[0].view(-1, *data_shape), 8)
save_image(x, fig_filename, nrow=nb)
default='./experiments/model-cifar-Resnet18',
help='directory of model for saving checkpoint')
parser.add_argument('--save-freq',
'-s',
default=50,
type=int,
metavar='N',
help='save frequency')
parser.add_argument('--coeff', type=float, default=0.99)
args = parser.parse_args()
# settings
model_dir = args.model_dir
utils.makedirs(model_dir)
logger = utils.get_logger(logpath=os.path.join(model_dir, 'logs'),
filepath=os.path.abspath(__file__))
logger.info(args)
use_cuda = not args.no_cuda and torch.cuda.is_available()
if use_cuda:
torch.backends.cudnn.benchmark = True
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 4, 'pin_memory': True} if use_cuda else {}
ACTIVATION_FNS = {
'identity': base_layers.Identity,
'relu': torch.nn.ReLU,
'tanh': torch.nn.Tanh,
def __init__(self, save_adj_name, temperature, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._model_kwargs = kwargs.get('model')
self._train_kwargs = kwargs.get('train')
self.temperature = float(temperature)
self.opt = self._train_kwargs.get('optimizer')
self.max_grad_norm = self._train_kwargs.get('max_grad_norm', 1.)
self.ANNEAL_RATE = 0.00003
self.temp_min = 0.1
self.save_adj_name = save_adj_name
self.epoch_use_regularization = self._train_kwargs.get('epoch_use_regularization')
self.num_sample = self._train_kwargs.get('num_sample')
# logging.
self._log_dir = self._get_log_dir(kwargs)
self._writer = SummaryWriter('runs/' + self._log_dir)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir, __name__, 'info.log', level=log_level)
# data set
self._data = utils.load_dataset(**self._data_kwargs)
self.standard_scaler = self._data['scaler']
### Feas
if self._data_kwargs['dataset_dir'] == 'data/METR-LA':
df = pd.read_hdf('./data/metr-la.h5')
elif self._data_kwargs['dataset_dir'] == 'data/PEMS-BAY':
df = pd.read_hdf('./data/pems-bay.h5')
else:
df = pd.read_csv('./data/pmu_normalized.csv', header=None)
df = df.transpose()
num_samples = df.shape[0]
num_train = round(num_samples * 0.7)
df = df[:num_train].values
scaler = utils.StandardScaler(mean=df.mean(), std=df.std())
train_feas = scaler.transform(df)
self._train_feas = torch.Tensor(train_feas).to(device)
print(self._train_feas.shape)
k = self._train_kwargs.get('knn_k')
knn_metric = 'cosine'
from sklearn.neighbors import kneighbors_graph
g = kneighbors_graph(train_feas.T, k, metric=knn_metric)
g = np.array(g.todense(), dtype=np.float32)
self.adj_mx = torch.Tensor(g).to(device)
self.num_nodes = int(self._model_kwargs.get('num_nodes', 1))
self.input_dim = int(self._model_kwargs.get('input_dim', 1))
self.seq_len = int(self._model_kwargs.get('seq_len')) # for the encoder
self.output_dim = int(self._model_kwargs.get('output_dim', 1))
self.use_curriculum_learning = bool(
self._model_kwargs.get('use_curriculum_learning', False))
self.horizon = int(self._model_kwargs.get('horizon', 1)) # for the decoder
# setup model
GTS_model = GTSModel(self.temperature, self._logger, **self._model_kwargs)
self.GTS_model = GTS_model.cuda() if torch.cuda.is_available() else GTS_model
self._logger.info("Model created")
self._epoch_num = self._train_kwargs.get('epoch', 0)
if self._epoch_num > 0:
self.load_model()
parser.add_argument('--val-freq', type=int, default=500)
parser.add_argument('--brute-val', action='store_true', default=False)
parser.add_argument('--log-freq', type=int, default=100)
parser.add_argument('--train-est-freq', type=int, default=None)
args = parser.parse_args()
args.val_batch_size = args.val_batch_size if args.val_batch_size else args.batch_size
args.test_batch_size = args.test_batch_size if args.test_batch_size else args.batch_size
args.train_est_freq = args.train_est_freq if args.train_est_freq else args.log_freq
args.data_root = args.data_root if args.data_root else datasets.root
save_path = os.path.join(args.save, 'checkpt.pth')
log_path = os.path.join(args.save, 'logs')
# Logger
utils.makedirs(args.save)
logger = utils.get_logger(logpath=log_path)
logger.info(args)
if args.fp64:
torch.set_default_dtype(torch.float64)
# noinspection PyPep8Naming
def batch_iter(X, batch_size=args.batch_size, shuffle=False):
"""
X: feature tensor (shape: num_instances x num_features)
"""
if shuffle:
idxs = torch.randperm(X.shape[0])
else:
idxs = torch.arange(X.shape[0])
def __init__(self, is_training=True, **kwargs):
self._kwargs = kwargs
self._data_kwargs = kwargs.get('data')
self._train_kwargs = kwargs.get('train')
self._test_kwargs = kwargs.get('test')
self._model_kwargs = kwargs.get('model')
self._alg_name = self._kwargs.get('alg')
# data args
self._raw_dataset_dir = self._data_kwargs.get('raw_dataset_dir')
self._test_size = self._data_kwargs.get('test_size')
# logging.
self._log_dir = self._get_log_dir(kwargs)
log_level = self._kwargs.get('log_level', 'INFO')
self._logger = utils.get_logger(self._log_dir,
__name__,
'info.log',
level=log_level)
self._logger.info(kwargs)
# Model's Args
self._model_type = self._model_kwargs.get('model_type')
self._verified_percentage = self._model_kwargs.get(
'verified_percentage')
self._rnn_units = self._model_kwargs.get('rnn_units')
self._seq_len = self._model_kwargs.get('seq_len')
self._horizon = self._model_kwargs.get('horizon')
self._input_dim = self._model_kwargs.get('input_dim')
self._input_shape = (self._seq_len, self._input_dim)
self._output_dim = self._model_kwargs.get('output_dim')
self._nodes = self._model_kwargs.get('num_nodes')
self._num_rnn_layers = self._model_kwargs.get('num_rnn_layers')
# Train's args
self._drop_out = self._train_kwargs.get('dropout')
self._epochs = self._train_kwargs.get('epochs')
self._batch_size = self._data_kwargs.get('batch_size')
self._optimizer = self._train_kwargs.get('optimizer')
# Test's args
self._run_times = self._test_kwargs.get('run_times')
# Load data
if self._model_type == 'ed' or self._model_type == 'encoder_decoder':
self._data = utils.load_dataset_lstm_ed(
seq_len=self._seq_len,
horizon=self._horizon,
input_dim=self._input_dim,
output_dim=self._output_dim,
raw_dataset_dir=self._raw_dataset_dir,
r=self._verified_percentage,
p=self._test_size,
**kwargs)
else:
raise RuntimeError("Model must be lstm or encoder_decoder")
self.callbacks_list = []
self._checkpoints = ModelCheckpoint(self._log_dir + "best_model.hdf5",
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto',
period=1)
self.callbacks_list = [self._checkpoints]
self._earlystop = EarlyStopping(
monitor='val_loss',
patience=self._train_kwargs.get('patience'),
verbose=1,
mode='auto')
self.callbacks_list.append(self._earlystop)
self._time_callback = TimeHistory()
self.callbacks_list.append(self._time_callback)
self.model = self._model_construction(is_training=is_training)
for i, horizon in enumerate(n_forwards):
rmse = masked_rmse_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mape = masked_mape_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
mae = masked_mae_np(preds=y_predicts[i].as_matrix(),
labels=y_test.as_matrix(),
null_val=0)
line = 'VAR\t%d\t%.2f\t%.2f\t%.2f' % (horizon, rmse, mape * 100, mae)
logger.info(line)
def main(args):
traffic_reading_df = pd.read_hdf(args.traffic_reading_filename)
eval_static(traffic_reading_df)
eval_historical_average(traffic_reading_df, period=3)
eval_var(traffic_reading_df, n_lags=3)
if __name__ == '__main__':
logger = utils.get_logger('data/model', 'Baseline')
parser = argparse.ArgumentParser()
parser.add_argument('--traffic_reading_filename',
default="data/metr-la.h5",
type=str,
help='Path to the traffic Dataframe.')
args = parser.parse_args()
main(args)
parser.add_argument('--nworkers', type=int, default=2)
#parser.add_argument('--nworkers', type=int, default=4)
#parser.add_argument('--nworkers', type=int, default=4)
parser.add_argument('--print-freq', help='Print progress every so iterations', type=int, default=20)
parser.add_argument('--vis-freq', help='Visualize progress every so iterations', type=int, default=500)
args = parser.parse_args()
# Random seed
if args.seed is None:
args.seed = np.random.randint(100000)
# logger
utils.makedirs(args.save)
logger = utils.get_logger(logpath=os.path.join(args.save, 'logs'), filepath=os.path.abspath(__file__))
logger.info(args)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
if device.type == 'cuda':
logger.info('Found {} CUDA devices.'.format(torch.cuda.device_count()))
for i in range(torch.cuda.device_count()):
props = torch.cuda.get_device_properties(i)
logger.info('{} \t Memory: {:.2f}GB'.format(props.name, props.total_memory / (1024**3)))
else:
logger.info('WARNING: Using device {}'.format(device))
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if device.type == 'cuda':
import numpy as np
import torch
from bm_sequential import get_test_dataset as get_dataset
from ctfp_tools import build_augmented_model_tabular
from ctfp_tools import parse_arguments
from ctfp_tools import run_ctfp_model as run_model
from train_misc import (
create_regularization_fns, )
from train_misc import set_cnf_options
torch.backends.cudnn.benchmark = True
if __name__ == "__main__":
args = parse_arguments()
logger = utils.get_logger(logpath=os.path.join(args.save, "logs_test"),
filepath=os.path.abspath(__file__))
if args.layer_type == "blend":
logger.info(
"!! Setting time_length from None to 1.0 due to use of Blend layers."
)
args.time_length = 1.0
logger.info(args)
# get deivce
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if args.use_cpu:
device = torch.device("cpu")
cvt = lambda x: x.type(torch.float32).to(device, non_blocking=True)
# load dataset