def __init__(self,
input_size: Tuple[int, int],
input_dim: int,
hidden_dim: List[int],
kernel_size: List[Tuple[int, int]],
batch_first: bool = True,
bias: bool = True,
decoding_steps: int = -1):
super(ConvLSTMAutoencoder, self).__init__()
self.decoding_steps = decoding_steps
self.input_size = input_size
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.kernel_size = kernel_size
self.batch_first = batch_first
self.num_layers = len(hidden_dim)
self.encoder = ConvLSTM(input_size=input_size,
input_dim=input_dim,
hidden_dim=hidden_dim,
kernel_size=kernel_size,
num_layers=self.num_layers,
batch_first=False,
bias=bias,
mode=ConvLSTM.SEQUENCE)
# reverse the order of hidden dimensions and kernels
decoding_hidden_dim = list(reversed(hidden_dim))
decoding_kernel_size = list(reversed(kernel_size))
decoding_hidden_dim.append(
input_dim
) # NOTE: we need a num_of_decoding_layers = num_of_encoding_layers+1
decoding_kernel_size.append(
(1, 1)) # so we add a 1x1 ConvLSTM as last decoding layer
self.input_reconstruction = ConvLSTM(input_size=input_size,
input_dim=input_dim,
hidden_dim=decoding_hidden_dim,
kernel_size=decoding_kernel_size,
num_layers=self.num_layers + 1,
batch_first=False,
bias=bias,
mode=ConvLSTM.STEP_BY_STEP)
self.future_prediction = ConvLSTM(input_size=input_size,
input_dim=input_dim,
hidden_dim=decoding_hidden_dim,
kernel_size=decoding_kernel_size,
num_layers=self.num_layers + 1,
batch_first=False,
bias=bias,
mode=ConvLSTM.STEP_BY_STEP)
def __init__(self,pretrainedModel):
super(PolygonRNN, self).__init__()
self.VGG = TruncatedVGG(pretrainedModel)
#First VGG block 56 x 56 x 128
self.mp1 = nn.MaxPool2d(2,2) #28 x 28 x 128
self.conv1 = nn.Conv2d(128, 128, kernel_size=3,stride=1,padding=1) #28 x 28 x 128
#Second VGG block 28 x 28 x 256
self.conv2 = nn.Conv2d(256, 128, kernel_size=3,stride=1,padding=1) #28 x 28 x 128
#Third VGG block 28 x 28 x 512
self.conv3 = nn.Conv2d(512, 128, kernel_size=3,stride=1,padding=1) #28 x 28 x 128
#Fourth VGG block 14 x 14 x 512
self.conv4 = nn.Conv2d(512, 128, kernel_size=3,stride=1,padding=1) #14 x 14 x 128
self.up4 = nn.Upsample(scale_factor=2, mode='bilinear') # 28 x 28 x 128
#Fused VGG BLock 28 x 28 x 512
self.convFused = nn.Conv2d(512, 128, kernel_size=3,stride=1,padding=1) # 28 x 28 x 128
self.ConvLSTM = ConvLSTM(input_size=(28,28),
input_dim=128,
hidden_dim=[16,1],
kernel_size=(3,3),
num_layers=2,
bias=True,
batch_first=True)
def __init__(self, args):
super().__init__()
self.args = args
self.lstm = ConvLSTM(input_size=(args.image_size // 32,
args.image_size // 32),
kernel_size=(3, 3),
input_dim=args.nfeatures,
hidden_dim=args.nfeatures // 4,
num_layers=2,
batch_first=True,
return_all_layers=False)
self.fc = nn.Linear(args.nfeatures // 4, args.nclasses)
def __init__(self):
super(sequence_to_one, self).__init__()
self.model = ConvLSTM(input_size=(32, 32),
input_dim=3,
hidden_dim=[64, 64, 128],
kernel_size=(3, 3),
num_layers=3,
batch_first=True,
bias=True,
return_all_layers=False)
self.fc1 = nn.Linear(128*32*32, 1024).cuda()
self.fc2 = nn.Linear(1024, 1).cuda()
def _decode(decoder: ConvLSTM, last_frame: Tensor,
representation: HiddenState, steps: int) -> Tensor:
decoded_sequence = []
h_n, c_n = representation
h_0, c_0 = decoder.init_hidden(last_frame.size(0))
h_0[0], c_0[0] = h_n, c_n
state = (h_0, c_0)
output = last_frame
for t in range(steps):
output, state = decoder(output, state)
decoded_sequence.append(output)
return torch.stack(decoded_sequence, dim=0)
def __init__(self, num_classes):
super(ConvLstmNet, self).__init__()
self.features = ConvLSTM(input_dim=3,
hidden_dim=[16],
kernel_size=(3, 3),
num_layers=1,
batch_first=True,
bias=True,
return_all_layers=False)
self.avgpool = nn.AdaptiveAvgPool2d((3, 3))
self.classifier = nn.Sequential(
nn.Dropout(),
nn.Linear(16 * 3 * 3, 128),
nn.ReLU(inplace=True),
nn.Linear(128, num_classes),
)
def __init__(self):
super().__init__()
self.enc_1 = Vgg()
self.conv_lstm = ConvLSTM(128,
128, (3, 3),
1,
batch_first=True,
return_all_layers=True)
self.conv_1 = nn.Conv2d(128, 128, (7, 7))
self.conv_2 = nn.Conv2d(128, 128, (7, 7))
self.de_cell1 = nn.LSTMCell(128, 128)
# self.de_cell2 = nn.LSTMCell(128, 128)
self.fc_in_lstm = nn.Linear(35, 128)
self.fc_os = nn.Linear(128, 35)
self.fc_r = nn.Linear(128, 82)
self.attention = torch.nn.MultiheadAttention(128, 1)
def main(train_dataset=None, validation_dataset=None, test_dataset=None, epoch=30, batch_size=128,
logdir=None, run=1, variation=1, checkpoint=None):
"""HAR Model Trainer
Train the `ConvLSTM` model with several `variation` of `ConvLSTMHyperparameter`. The model is
trained with dataset from `train_dataset`, validated with dataset from `validation_dataset` and
tested with dataset from `test_dataset`.
Training summary and checkpoints is saved to `logdir`. A log directory is created for each
variation, started from number provided to `run`.
To restore a checkpoint before training or testing, provide the path to `checkpoint`.
Args:
- `train_dataset`: path to train dataset
- `validation_dataset`: path to validation dataset
- `test_dataset`: path to test dataset
- `epoch`: number of epoch to train
- `batch_size`: mini batch size used for training
- `logdir`: path to save checkpoint and summary
- `run`: number of run for the first variation, used for log directory naming
- `variation`: number of hyperparameter variation
- `checkpoint`: checkpoint path to restore
"""
hyperparameters = convlstm_hyperparamter(variation)
for i, hyperparameter in enumerate(hyperparameters):
run_logdir = os.path.join(logdir, 'run' + str(i + run))
model = ConvLSTM(hyperparameter, run_logdir)
print('Run %d/%d' % (i + 1, variation))
print_hyperparameter_notes(hyperparameter)
write_hyperparameter_notes(hyperparameter, run_logdir)
if train_dataset and validation_dataset:
train_data = load(train_dataset, NUM_TARGET, WINDOW_SIZE)
validation_data = load(validation_dataset, NUM_TARGET, WINDOW_SIZE)
if train_data.data.any() and validation_data.data.any():
model.train(train_data, validation_data, epoch, batch_size, checkpoint)
if test_dataset:
test_data = load(test_dataset, NUM_TARGET, WINDOW_SIZE)
if test_data.data.any():
prediction = model.test(test_data, batch_size, checkpoint)
model.confusion_matrix(prediction, test_data.target)
tf.reset_default_graph()
def __init__(self, opt):
super(ConvLSTMAE, self).__init__()
#
self.opt = opt # 配置参数
#
self.Conv3 = nn.Sequential(
# TODO：图片的channel不是1，但是作者网络设置是1，有冲突？
nn.Conv2d(self.opt.channel, 128, 7, 4, 3),
nn.ReLU(),
nn.Conv2d(128, 256, 5, 2, 2),
nn.ReLU(),
nn.Conv2d(256, 512, 3, 2, 1),
nn.ReLU(),
)
# 下面的ConvLSTM和本论文作者的code有一些差异，等最后再调节这个！
height, width = 15, 15 # TODO 每次都要根据Conv3输出的x的w,h来设置
self.ConvLSTM = ConvLSTM(
input_size=(height, width), # Conv3后tensor的w,h
input_dim=512, # Conv3后tensor的channel
hidden_dim=[64, 64, 128],
kernel_size=(3, 3),
num_layers=3,
batch_first=True,
bias=False, # 作者论文
return_all_layers=False # 只返回最后一层
)
self.Deconv3_1 = nn.Sequential(
nn.ConvTranspose2d(128, 256, 3, 2,
1), # TODO input_channel根据ConvLSTM的output改
nn.ReLU(),
nn.ConvTranspose2d(256, 128, 5, 2, 2),
nn.ReLU(),
nn.ConvTranspose2d(128, self.opt.channel, 7, 4, 3),
nn.Tanh(),
)
self.Deconv3_2 = nn.Sequential(
nn.ConvTranspose2d(128, 256, 3, 2,
1), # TODO input_channel根据ConvLSTM的output改
nn.ReLU(),
nn.ConvTranspose2d(256, 128, 5, 2, 2),
nn.ReLU(),
nn.ConvTranspose2d(128, self.opt.channel, 7, 4, 3),
nn.Tanh(),
)
def __init__(self):
super(Net05, self).__init__()
#self.conv1 = ConvLSTM(input_channels=1, hidden_channels=[64, 32, 32], kernel_size=3, step=5,
# effective_step=[4]).cuda()
self.maxpool = nn.MaxPool3d((1, 5, 5), stride=(1, 3, 3))
self.BD = nn.BatchNorm2d(16)
self.drop = nn.Dropout(p=0.3)
self.conv1 = ConvLSTM(input_size=(74, 74),
input_dim=3,
hidden_dim=[64, 32, 16],
kernel_size=(3, 3),
num_layers=3,
batch_first=True)
self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2)
self.fc1 = nn.Linear(5 * 16 * 9 * 9, 120)
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84, 2)
def __init__(self):
super(CNNLSTM, self).__init__()
self.input_size = input_size
self.input_dim = input_dim
self.hidden_dim = [lstm_channel for _ in range(frame_num)]
self.num_layers = frame_num
self.down_sample = DownSample(down_in_dim, down_out_dim)
self.convlstm = ConvLSTM(
input_size=self.input_size,
input_dim=self.input_dim,
hidden_dim=self.hidden_dim,
kernel_size=(3, 3),
num_layers=self.num_layers,
batch_first=True,
bias=True,
return_all_layers=False
)
self.up_sample = UpSample(up_in_dim, up_out_dim)
def main():
model_name = "convlstm"
system = "ccw" # ccw, gf
system_dir = {"ccw":"concentric_circle_wave", "gf":"global_flow"}
result_dir = os.path.join("results", system_dir[system], "convlstm")
config = configparser.ConfigParser()
config.read("config_{}.ini".format(system))
if not os.path.exists(result_dir):
os.mkdir(result_dir)
# shutil.copy("config_{}.ini".format(system), result_dir)
## Set seed and cuda
seed = 128
print("Set seed {}.".format(seed))
cuda = torch.cuda.is_available()
gpu = int(config.get("train", "gpu"))
if cuda:
print("cuda is available")
device = torch.device('cuda', gpu)
else:
print("cuda is not available")
device = torch.device('cpu')
torch.manual_seed(seed)
if cuda:
torch.cuda.manual_seed(seed)
np.random.seed(seed)
# cuda = False
# device = torch.device('cpu')
# np.random.seed(seed)
# torch.autograd.set_detect_anomaly(True)
## Read data and set parameters
train_data = np.load(config.get("data", "path")).astype(np.float32) # T x h x w
true_data = np.load(config.get("data", "true_path")).astype(np.float32)
timesteps = int(config.get("data", "timesteps"))
width = int(config.get("data", "width"))
height = int(config.get("data", "height"))
loss_name = config.get("network", "loss")
#n_layers = int(config.get("network", "n_layers"))
step = int(config.get("network", "step"))
effective_step = [int(i) for i in config.get("network", "effective_step").split(",")]
input_channels = int(config.get("network", "input_channels"))
kernel_size = tuple([int(i) for i in config.get("network", "kernel_size").split(",")])
n_channels = [int(i) for i in config.get("network", "n_channels").split(",")]
n_layers = len(n_channels)
batch_norm = bool(config.get("network", "batch_norm"))
effective_layers = [int(i) for i in config.get("network", "effective_layers").split(",")]
num_epochs = int(config.get("train", "num_epochs"))
batch_size = int(config.get("train", "batch_size"))
optimizer_name = config.get("train", "optimizer")
init_lr = float(config.get("train", "init_lr"))
decay_rate = float(config.get("train", "decay_rate"))
decay_steps = float(config.get("train", "decay_steps"))
train_steps = int(config.get("train", "train_steps"))
test_steps = int(config.get("train", "test_steps"))
prediction_steps = int(config.get("train", "prediction_steps"))
display_steps = int(config.get("logs", "display_steps"))
save_steps = int(config.get("logs", "save_steps"))
## Read model
model = ConvLSTM((height, width), input_channels, n_channels, kernel_size, n_layers, effective_layers, batch_norm, device=device).to(device)
if loss_name == "MSE":
loss_fn = nn.MSELoss()
elif loss_name == "CE":
loss_fn = nn.CrossEntropyLoss()
if cuda:
cudnn.benchmark = True
if optimizer_name == "Adam":
optimizer = torch.optim.Adam(model.parameters(), lr=init_lr)
elif optimizer_name == "RMSprop":
optimizer = torch.optim.RMSprop(model.parameters(), lr=init_lr)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=decay_steps, gamma=decay_rate)
# Define functions
def train(epoch):
model.train()
epoch_loss = 0
data = Variable(torch.from_numpy(train_data[:train_steps-1])).unsqueeze(1).unsqueeze(1).to(device) # T x bs(=1) x c(=1) x h x w
# forward + backward + optimize
optimizer.zero_grad()
outputs, _ = model(data)
loss = loss_fn(outputs.squeeze(), torch.from_numpy(train_data[1:train_steps]).to(device))
loss.backward()
optimizer.step()
epoch_loss = loss.item()
if epoch%display_steps==0:
print_contents = "Train Epoch: [{}/{}]".format(epoch, num_epochs)
print_contents += "\t {}: {:.6f}".format(
loss_name,
epoch_loss)
print(print_contents)
return epoch_loss
def test(epoch):
"""uses test data to evaluate likelihood of the model"""
model.eval()
epoch_loss = 0
data = Variable(torch.from_numpy(train_data[train_steps:train_steps+test_steps-1])).unsqueeze(1).unsqueeze(1).to(device) # T x bs(=1) x c(=1) x w x h
# forward + backward + optimize
optimizer.zero_grad()
outputs, _ = model(data)
loss = loss_fn(outputs.squeeze(), torch.from_numpy(train_data[train_steps+1:train_steps+test_steps]).to(device))
loss.backward()
optimizer.step()
epoch_loss = loss.item()
if epoch%display_steps==0:
print_contents = "====> Test set loss:"
print_contents += " {} = {:.4f}".format(loss_name,
epoch_loss)
print(print_contents)
return epoch_loss
def prediction(epoch):
"""n-step prediction"""
model.eval()
loss = np.zeros((2, prediction_steps))
output = np.zeros((prediction_steps, train_data.shape[1], train_data.shape[2]))
data = Variable(torch.from_numpy(train_data[:train_steps-1].squeeze()))
data = data.unsqueeze(1).unsqueeze(1).to(device) # T x bs(=1) x c(=1) x h x w
outputs, last_state_list = model(data)
#prev_state = outputs[-1].view(1,1,1,height,width) # T(=1) x bs(=1) x c(=1) x h x w
prev_state = Variable(torch.from_numpy(train_data[train_steps])).unsqueeze(0).unsqueeze(0).unsqueeze(0).to(device)
for i in range(prediction_steps):
prev_state, last_state_list = model(prev_state, last_state_list)
loss[0,i] = mean_squared_error(prev_state.squeeze().cpu().detach().numpy(), train_data[train_steps+i])
loss[1,i] = mean_squared_error(prev_state.squeeze().cpu().detach().numpy(), true_data[train_steps+i])
output[i] = prev_state.squeeze().cpu().detach().numpy()
if epoch%display_steps==0:
print_contents = "===> Prediction loss:\n"
for i in range(prediction_steps):
print_contents += "{} step forecast {}: {}\n".format(i+1, loss_name, loss[0,i])
print(print_contents)
#print("output", output.shape, output.min(), output.max())
return loss, output
## Train model
def execute():
train_loss = np.zeros(num_epochs)
test_loss = np.zeros(num_epochs)
prediction_loss = np.zeros((num_epochs, 2, prediction_steps))
outputs = np.zeros((num_epochs//save_steps, prediction_steps, train_data.shape[1], train_data.shape[2]))
start_time = time.time()
for epoch in range(1, num_epochs + 1):
# training + testing
_train_loss = train(epoch)
_test_loss = test(epoch)
_prediction_loss = prediction(epoch)
scheduler.step()
# substitute losses for array
train_loss[epoch-1] = _train_loss
test_loss[epoch-1] = _test_loss
prediction_loss[epoch-1], outputs[(epoch-1)//save_steps] = _prediction_loss
# duration
duration = int(time.time() - start_time)
second = int(duration%60)
remain = int(duration//60)
minute = int(remain%60)
hour = int(remain//60)
print("Duration: {} hour, {} min, {} sec.".format(hour, minute, second))
remain = (num_epochs - epoch) * duration / epoch
second = int(remain%60)
remain = int(remain//60)
minute = int(remain%60)
hour = int(remain//60)
print("Estimated Remain Time: {} hour, {} min, {} sec.".format(hour,
minute,
second))
# saving model
if epoch % save_steps == 0:
torch.save(model.state_dict(), os.path.join(result_dir,
'state_dict_'+str(epoch)+'.pth'))
torch.save(optimizer.state_dict(), os.path.join(result_dir,
'adam_state_dict_'+str(epoch)+'.pth'))
print('Saved model to state_dict_'+str(epoch)+'.pth')
# np.save(os.path.join(result_dir, "train_loss.npy"), train_loss)
# np.save(os.path.join(result_dir, "test_loss.npy"), test_loss)
np.save(os.path.join(result_dir, "prediction_loss.npy"), prediction_loss)
np.save(os.path.join(result_dir, "convlstm_mse.npy"), prediction_loss[:,1])
# np.save(os.path.join(result_dir, "prediction.npy"), outputs)
# plot loss for train and test
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
ax.plot(range(epoch), train_loss[:epoch],
label="train")
ax.plot(range(epoch), test_loss[:epoch],
label="test")
ax.set_xlabel("epoch")
ax.set_ylabel(loss_name)
ax.legend()
fig.savefig(os.path.join(result_dir, "loss.png"),
bbox_inches="tight")
ax.set_yscale("log")
fig.savefig(os.path.join(result_dir, "log_loss.png"),
bbox_inches="tight")
# plot prediction loss
fig, ax = plt.subplots(1, 1, figsize=(5, 5))
for i in range(save_steps, epoch+1, save_steps):
ax.plot(range(train_steps, train_steps+prediction_steps), prediction_loss[i-1,0,:],
label="epoch={}".format(i))
ax.set_xlabel("timestep")
ax.set_ylabel(loss_name)
ax.legend()
fig.savefig(os.path.join(result_dir, "prediction_loss.png"),
bbox_inches="tight")
ax.set_yscale("log")
fig.savefig(os.path.join(result_dir, "log_prediction_loss.png"),
bbox_inches="tight")
## Excute
# measure for culabas rutine error
execute()
seq_size = 16
transform = transforms.Compose([RandomNoiseWithGT(), transforms.ToTensor()])
raw_data = dset.MNIST("../autoencoderMNIST/MNIST",
download=True,
transform=transform)
dloader = torch.utils.data.DataLoader(raw_data,
batch_size=batch_size,
shuffle=True,
drop_last=True)
encoder = ConvLSTM(
input_size=(7, 7),
input_dim=1,
hidden_dim=[8, 16, 32, 64],
kernel_size=(3, 3),
num_layers=3,
)
decoder = ConvLSTM(
input_size=(7, 7),
input_dim=64,
hidden_dim=[32, 16, 8, 1],
kernel_size=(3, 3),
num_layers=3,
)
encoder.cuda()
decoder.cuda()
b_size = 128
transform = transforms.Compose([transforms.ToTensor()])
raw_data = dset.MNIST("../autoencoderMNIST/MNIST",
download=False,
transform=transform)
dloader = torch.utils.data.DataLoader(raw_data,
batch_size=b_size,
shuffle=True,
drop_last=True)
encoder = ConvLSTM(
input_size=(28, 28),
input_dim=1,
hidden_dim=[2, 1],
kernel_size=(3, 3),
num_layers=2,
)
decoder = ConvLSTM(
input_size=(28, 28),
input_dim=1,
hidden_dim=[2, 1],
kernel_size=(3, 3),
num_layers=2,
)
encoder.cuda()
decoder.cuda()
transforms_=transforms_)
#train_loader = DataLoader(train_dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_cpu)
train_loader = DataLoader(train_dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_cpu)
test_loader = DataLoader(test_dataset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.n_cpu)
model = ConvLSTM(opt.in_channels,
2 * opt.in_channels,
opt.kernel_size,
opt.num_layers,
batch_first=False,
bias=True,
return_all_layers=False)
#model = PredictorLSTM(opt.input_size, opt.hidden_size, opt.num_layers, opt.out_size) # 27 *8
use_gpu = True if torch.cuda.is_available() else False
if use_gpu:
model = model.cuda()
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=opt.lr, betas=(opt.b1, opt.b2))
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=opt.milestones,
gamma=0.5)
header = ['epoch/total_epoch', 'test_mse']
word_to_ix = {"hello": 0, "world": 1}
#embeds = nn.Embedding(2, 5) # 2 words in vocab, 5 dimensional embeddings
embeds = nn.Embedding(2,
feat_dim_h) # 2 words in vocab, 7 dimensional embeddings
lookup_tensor = torch.LongTensor([word_to_ix["hello"]])
hello_embed = embeds(autograd.Variable(lookup_tensor))
lookup_tensor = torch.LongTensor([word_to_ix["world"]])
world_embed = embeds(autograd.Variable(lookup_tensor))
print("Hello Emb:", hello_embed)
encoder = ConvLSTM(
input_size=(feat_dim_h, feat_dim_w),
input_dim=feat_dim_chan + feat_dim_h,
hidden_dim=hidden_size,
kernel_size=(3, 3),
num_layers=2,
)
encoder.cuda()
crit = nn.MSELoss() #nn.BCELoss()
crit.cuda()
threshold = nn.Threshold(0., 0.0)
params = list(encoder.parameters())
optimizer = optim.Adam(params, lr=0.001)
s = 1
tester = test.TesterMnist(out_dir=test_out_dir)
#--------model---------
cnn_encoder = ConvEncoder(hidden_dim)
cnn_decoder = ConvDecoder(b_size=b_size,inp_dim=hidden_dim)
cnn_encoder.cuda()
cnn_decoder.cuda()
lstm_encoder = ConvLSTM(
input_size=(hidden_spt,hidden_spt),
input_dim=hidden_dim,
hidden_dim=lstm_dims,
kernel_size=(3,3),
num_layers=3,
peephole=True,
batchnorm=False,
batch_first=True,
activation=F.tanh
)
lstm_decoder = ConvLSTM(
input_size=(hidden_spt,hidden_spt),
input_dim=hidden_dim,
hidden_dim=lstm_dims,
kernel_size=(3,3),
num_layers=3,
peephole=True,
batchnorm=False,
batch_first=True,