def save_score_test(score):
dec = Decoder(config)
dec.save_score('test_score', score.values)
def set_decoder(self):
self.decoder = Decoder(self.argv)
def test_autoencoder():
kl_weight = 1.0
reconstr_weight = 1.0
learning_rate = 0.001
mb_size = 64
use_cuda = torch.cuda.is_available()
torch.manual_seed(0)
np.random.seed(0)
device = torch.device("cuda" if use_cuda else "cpu")
train_data = pickle.load(open('data/pusher_relabeled.pkl',
'rb'))[:450].reshape(450 * 50, 1, 64, 64,
3).astype(np.float32)
val_data = pickle.load(open('data/pusher_relabeled.pkl',
'rb'))[450:].reshape(50 * 50, 1, 64, 64,
3).astype(np.float32)
train_dataset = ObjDataset(train_data, device)
val_dataset = ObjDataset(val_data, device)
train_dataloader = DataLoader(train_dataset,
batch_size=mb_size,
shuffle=True,
num_workers=8)
val_dataloader = DataLoader(val_dataset,
batch_size=1,
shuffle=True,
num_workers=1)
enc = Encoder([3, 96, 96, 8], [0, 0, 0, 8], 8).to(device)
dec = Decoder([9, 96, 96, 3], 8).to(device)
params = {}
for (k, v) in enc.named_parameters():
params['enc.' + k.replace('__', '.')] = v
for (k, v) in dec.named_parameters():
params['dec.' + k.replace('__', '.')] = v
optimizer = optim.Adam(params.values(), lr=learning_rate)
logdir = 'pusher2_ae_kl__10000_' + time.strftime("%d-%m-%Y_%H-%M")
n_validation_samples = 5
eps = 1e-20
enc.train()
dec.train()
model_forward = lambda ims_tensor: ae_forward(enc, dec, ims_tensor)
#'data/pusher2_ae_kl_0_09-12-2018_09-53/9/params.pkl' #lin dec
#'data/pusher2_ae_kl__10000_10-12-2018_01-54/6/params.pkl' #sparse lin dec
params = init_weights(
params,
file='data/pusher2_ae_kl__10000_10-12-2018_01-54/6/params.pkl',
device=device)
for epoch in range(10): #10 #30
for (train_ind, rollout) in tqdm(enumerate(train_dataloader)):
if train_ind >= 100:
break
rollout = rollout.to(device)
ims_tensor = rollout.reshape(-1, 3, 64, 64)
latent, samples, reconstr = model_forward(ims_tensor)
optimizer.zero_grad()
#sampled_beta = torch.sum(samples, dim=[0,2,3]) / samples.shape[0] / 64 / 64
#kl_loss = torch.mean((sampled_beta - 1/(64*64))**2)
sampled_beta = torch.mean(samples)
kl_loss = torch.mean((sampled_beta - 1 / (64 * 64 * 8))**2)
reconstr_loss = torch.mean((ims_tensor - reconstr)**2)
kl_weight = (epoch + 1) * 10000
loss = kl_weight * kl_loss + reconstr_weight * reconstr_loss
loss.backward()
optimizer.step()
if epoch % 1 == 0:
print(epoch,
kl_weight * kl_loss.detach().cpu().numpy(),
reconstr_weight * reconstr_loss.detach().cpu().numpy())
validate_model(logdir, epoch, val_dataloader, n_validation_samples,
model_forward, params, device)
print(epoch,
kl_weight * kl_loss.detach().cpu().numpy(),
reconstr_weight * reconstr_loss.detach().cpu().numpy())
validate_model(logdir, epoch, val_dataloader, n_validation_samples,
model_forward, params, device)
def train(log_dir, n_epochs, network_dict, index2token, mode, **kwargs):
onehot_words = kwargs['onehot_words']
word_pos = kwargs['word_pos']
sentence_lens_nchars = kwargs['sentence_lens_nchars']
sentence_lens_nwords = kwargs['sentence_lens_nwords']
vocabulary_size = kwargs['vocabulary_size']
max_char_len = kwargs['max_char_len']
onehot_words_val = kwargs['onehot_words_val']
word_pos_val = kwargs['word_pos_val']
sentence_lens_nchars_val = kwargs['sentence_lens_nchars_val']
batch_size = kwargs['batch_size']
input_size = vocabulary_size
hidden_size = kwargs['hidden_size']
decoder_dim = kwargs['decoder_dim']
decoder_units_p3 = kwargs['decoder_units_p3']
network_dict['input_size'] = input_size
# prepping permutation matrix for all instances seperately
perm_mat, max_lat_word_len, lat_sent_len_list = train_helper.prep_perm_matrix(batch_size=batch_size,
word_pos_matrix=word_pos,
max_char_len=max_char_len)
onehot_words = np.reshape(onehot_words, newshape=[-1, batch_size, max_char_len, vocabulary_size])
word_pos = np.reshape(word_pos, newshape=[-1, batch_size, max_char_len])
sentence_lens_nchars = np.reshape(sentence_lens_nchars, newshape=[-1, batch_size])
lat_sent_len_list = np.reshape(lat_sent_len_list, [-1, batch_size])
# shaping for validation set
batch_size_val = batch_size
n_valid = np.shape(onehot_words_val)[0]
r = n_valid % batch_size_val
n_valid_use = n_valid - r
onehot_words_val = np.reshape(onehot_words_val[0:n_valid_use, ...],
newshape=[-1, batch_size_val, max_char_len, vocabulary_size])
word_pos_val = np.reshape(word_pos_val[0:n_valid_use, ...], newshape=[-1, batch_size_val, max_char_len])
sentence_lens_nchars_val = np.reshape(sentence_lens_nchars_val[0:n_valid_use], newshape=[-1, batch_size_val])
perm_mat_val, _, lat_sent_len_list_val = train_helper.prep_perm_matrix(batch_size=batch_size_val,
word_pos_matrix=word_pos_val,
max_char_len=max_char_len,
max_word_len=max_lat_word_len)
lat_sent_len_list_val = np.reshape(np.reshape(lat_sent_len_list_val, -1)[0:n_valid_use],
newshape=[-1, batch_size_val])
# kl_mask
kl_mask, kl_mask_val = train_helper.kl_mask_prep(lat_sent_len_list, lat_sent_len_list_val, max_lat_word_len,
batch_size)
# logging
log_file = log_dir + "vaelog.txt"
logger = logging.getLogger('mVAE_log')
hdlr = logging.FileHandler(log_file)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)
# gaussian samples seed
global_sample = np.random.normal(size=[batch_size, decoder_dim])
word_samples = np.random.normal(size=[batch_size, max_lat_word_len, decoder_dim])
if mode == "new_model":
# placeholders
mask_kl_pl = tf.placeholder(name='mask_kl_pl', dtype=tf.float32, shape=[batch_size, max_lat_word_len])
sent_word_len_list_pl = tf.placeholder(name='sent_word_len_list_pl', dtype=tf.int32, shape=[batch_size])
perm_mat_pl = tf.placeholder(name='perm_mat_pl', dtype=tf.int32, shape=[batch_size, max_lat_word_len])
onehot_words_pl = tf.placeholder(name='onehot_words_pl', dtype=tf.float32,
shape=[batch_size, max_char_len, vocabulary_size])
word_pos_pl = tf.placeholder(name='word_pos_pl', dtype=tf.float32, shape=[batch_size, max_char_len])
sent_char_len_list_pl = tf.placeholder(name='sent_char_len_list_pl', dtype=tf.float32, shape=[batch_size])
mask_kl_pl_val = tf.placeholder(name='mask_kl_pl_val', dtype=tf.float32, shape=[batch_size, max_lat_word_len])
sent_word_len_list_pl_val = tf.placeholder(name='sent_word_len_list_pl_val', dtype=tf.int32,
shape=[batch_size])
perm_mat_pl_val = tf.placeholder(name='perm_mat_pl_val', dtype=tf.int32, shape=[batch_size, max_lat_word_len])
onehot_words_pl_val = tf.placeholder(name='onehot_words_pl_val', dtype=tf.float32,
shape=[batch_size, max_char_len, vocabulary_size])
word_pos_pl_val = tf.placeholder(name='word_pos_pl_val', dtype=tf.float32, shape=[batch_size, max_char_len])
sent_char_len_list_pl_val = tf.placeholder(name='sent_char_len_list_pl_val', dtype=tf.float32,
shape=[batch_size])
# step counter
global_step = tf.Variable(0, name='global_step', trainable=False)
encoder_k = encoder.Encoder(**network_dict)
word_state_out, mean_state_out, logsig_state_out = encoder_k.run_encoder(sentence_lens=sent_char_len_list_pl,
train=True, inputs=onehot_words_pl,
word_pos=word_pos_pl, reuse=None)
word_state_out.set_shape([max_char_len, batch_size, hidden_size])
mean_state_out.set_shape([max_char_len, batch_size, hidden_size])
logsig_state_out.set_shape([max_char_len, batch_size, hidden_size])
word_state_out_p = train_helper.permute_encoder_output(encoder_out=word_state_out, perm_mat=perm_mat_pl,
batch_size=batch_size, max_word_len=max_lat_word_len)
mean_state_out_p = train_helper.permute_encoder_output(encoder_out=mean_state_out, perm_mat=perm_mat_pl,
batch_size=batch_size, max_word_len=max_lat_word_len)
logsig_state_out_p = train_helper.permute_encoder_output(encoder_out=logsig_state_out, perm_mat=perm_mat_pl,
batch_size=batch_size, max_word_len=max_lat_word_len)
# Initialize decoder
arg_dict = {'decoder_p3_units': decoder_units_p3, 'encoder_dim': hidden_size, 'lat_word_dim': hidden_size,
'sentence_lens': None, 'global_lat_dim': hidden_size, 'batch_size': batch_size,
'max_num_lat_words': max_lat_word_len, 'decoder_units': decoder_dim,
'num_sentence_characters': max_char_len, 'dict_length': vocabulary_size}
decoder = Decoder(**arg_dict)
train_logits, global_latent_o, global_logsig_o, global_mu_o = decoder.run_decoder(
word_sequence_length=sent_word_len_list_pl, train=True, reuse=None, units_lstm_decoder=decoder_dim,
lat_words=word_state_out_p, units_dense_global=decoder_dim,
char_sequence_length=tf.cast(sent_char_len_list_pl, dtype=tf.int32))
train_logits = tf.identity(train_logits, name="train_logits")
# KL annealing parameters
shift = 5000
total_steps = np.round(np.true_divide(n_epochs, 16) * np.shape(onehot_words)[0], decimals=0)
# calculate cost
train_cost, reconstruction, kl_p3, kl_p1, kl_global, kl_p2, anneal_value, _ = decoder.calc_cost(eow_mask=None,
mask_kl=mask_kl_pl,
kl=True,
sentence_word_lens=sent_word_len_list_pl,
shift=shift,
total_steps=total_steps,
global_step=global_step,
global_latent_sample=global_latent_o,
global_logsig=global_logsig_o,
global_mu=global_mu_o,
predictions=train_logits,
true_input=onehot_words_pl,
posterior_logsig=logsig_state_out_p,
posterior_mu=mean_state_out_p,
post_samples=word_state_out_p,
reuse=None)
# clipping gradients
lr = 1e-4
opt = tf.train.AdamOptimizer(lr)
grads_t, vars_t = zip(*opt.compute_gradients(train_cost))
clipped_grads_t, grad_norm_t = tf.clip_by_global_norm(grads_t, clip_norm=5.0)
train_step = opt.apply_gradients(zip(clipped_grads_t, vars_t), global_step=global_step, name="train_step")
# regex = re.compile('[^a-zA-Z]')
# sum_grad_hist = [tf.summary.histogram(name=regex.sub('',str(j)),values=i) for i,j in zip(clipped_grads_t,vars_t) if i is not None]
# norm_grad = tf.summary.scalar(name='grad_norm',tensor=grad_norm_t)
# testing graph
word_state_out_val, mean_state_out_val, logsig_state_out_val = encoder_k.run_encoder(
sentence_lens=sent_char_len_list_pl_val, train=False, inputs=onehot_words_pl_val, word_pos=word_pos_pl_val,
reuse=True)
word_state_out_val.set_shape([max_char_len, batch_size_val, hidden_size])
mean_state_out_val.set_shape([max_char_len, batch_size_val, hidden_size])
logsig_state_out.set_shape([max_char_len, batch_size_val, hidden_size])
word_state_out_p_val = train_helper.permute_encoder_output(encoder_out=word_state_out_val,
perm_mat=perm_mat_pl_val, batch_size=batch_size_val,
max_word_len=max_lat_word_len)
mean_state_out_p_val = train_helper.permute_encoder_output(encoder_out=mean_state_out_val,
perm_mat=perm_mat_pl_val, batch_size=batch_size_val,
max_word_len=max_lat_word_len)
logsig_state_out_p_val = train_helper.permute_encoder_output(encoder_out=logsig_state_out_val,
perm_mat=perm_mat_pl_val,
batch_size=batch_size_val,
max_word_len=max_lat_word_len)
# decode test
test_logits, global_latent_o_val, global_logsig_o_val, global_mu_o_val = decoder.run_decoder(
word_sequence_length=sent_word_len_list_pl_val, train=False, reuse=True, units_lstm_decoder=decoder_dim,
lat_words=mean_state_out_p_val, units_dense_global=decoder.global_lat_dim,
char_sequence_length=tf.cast(sent_char_len_list_pl_val, dtype=tf.int32))
test_logits = tf.identity(test_logits, name="test_logits")
# test cost
test_cost = decoder.test_calc_cost(mask_kl=mask_kl_pl_val, sentence_word_lens=sent_word_len_list_pl_val,
posterior_logsig=logsig_state_out_p_val, post_samples=word_state_out_p_val,
global_mu=global_mu_o_val, global_logsig=global_logsig_o_val,
global_latent_sample=global_latent_o_val, posterior_mu=mean_state_out_p_val,
true_input=onehot_words_pl_val, predictions=test_logits)
def main(img_path='../../test_.png', base=256):
global s1, s2, num, pic, encoder, decoder, predictor
num += 1
encoder = Encoder(out_channels=30)
decoder = Decoder(out_channels=30)
predictor = Predictor(30)
load(encoder, decoder, predictor)
encoder.eval()
decoder.eval()
img = cv2.imread(img_path)
img = np.array(img)
x = transforms.ToTensor()(img)
# x = x[:, 0:128 * 10, 0:128 * 10]
x = x.unsqueeze(0)
# x = x[:, :, 0:128, 0:128]
b, c, h, w = x.shape
if x.shape[2] % base != 0 or x.shape[3] % base != 0:
x = pad(x, base)
b, c, H, W = x.shape
x.requires_grad = False
if torch.cuda.is_available():
encoder = encoder.cuda()
decoder = decoder.cuda()
x = x.cuda()
y = x.clone()
if torch.cuda.is_available():
pass
z = []
for i in range(0, x.shape[2] // base):
for j in range(0, x.shape[3] // base):
z.append(x[:, :, i * base:(i + 1) * base, j * base:(j + 1) * base])
z = torch.cat(z, 0)
patches = z.shape[0]
print(patches)
z[0:patches // 4, :, :, :] = run(z[0:patches // 4, :, :, :]).detach()
z[patches // 4:patches // 2, :, :, :] = run(z[patches // 4:patches //
2, :, :, :]).detach()
z[patches // 2:patches * 3 // 4, :, :, :] = run(
z[patches // 2:patches * 3 // 4, :, :, :]).detach()
z[patches * 3 // 4:patches, :, :, :] = run(z[patches * 3 //
4:patches, :, :, :]).detach()
"""
pic = torch.cat(pic, 0) * 255
print(pic.shape)
pic = pic.reshape(-1, 36*30, 3).int().numpy()
cv2.imwrite('test.png', pic)
"""
tot = 0
for i in range(0, x.shape[2] // base):
for j in range(0, x.shape[3] // base):
x[:, :, i * base:(i + 1) * base, j * base:(j + 1) * base] = z[tot]
tot += 1
l1 = Lp_Loss.Loss(p=1)
ss = pytorch_ssim.SSIM(window_size=11)
psnr = PSNR()
psnr.eval()
ss.eval()
x = torch.clamp(x, 0, 1)
s1 += ss(x, y)
s2 += psnr(x, y)
x = x.detach()
y = y.detach()
s1 = s1.detach()
s2 = s2.detach()
print(s1 / num, s2 / num)
x = torch.round(x * 255).int()
x = torch.abs(x)
x = x.detach().numpy()
x = np.array(x, dtype=np.uint8)
x = x.squeeze(0)
x = np.swapaxes(x, 0, 2)
x = np.swapaxes(x, 0, 1)
x = x[0:h, 0:w, :]
cv2.imwrite('./out__.png', x)
def __init__(self, model_pre_path):
self.encoder = Encoder(model_pre_path)
self.decoder = Decoder(model_pre_path)
from signal_processing import detect_frequency
from decoder import Decoder
import sys
fs = 44100
CHUNKSIZE = 128
SENSITIVITY = 1500
PACE = .05
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt16,
channels=1,
rate=fs,
input=True,
frames_per_buffer=CHUNKSIZE)
d = Decoder(PACE, SENSITIVITY, sys.stdout, .003)
timestamps = []
start_time = time()
while time() - start_time < 100:
timestamps.append(time())
data = stream.read(CHUNKSIZE)
np_data = np.fromstring(data, dtype=np.int16)
amplitude = detect_frequency(np_data, 880.0, 44100)
d.add_frame(amplitude, timestamps[-1])
#print('amplitude: {}'.format(amplitude))
deltas = [timestamps[i] - timestamps[i - 1] for i in range(1, len(timestamps))]
avg_delta = sum(deltas) / len(deltas)
print('Average Delta: {}'.format(avg_delta))
def __init__(self,
src_vocab_size,
tgt_vocab_size,
cue_vocab_size,
goal_vocab_size,
embed_size,
hidden_size,
padding_idx=None,
num_layers=1,
bidirectional=True,
attn_mode="mlp",
with_bridge=False,
tie_embedding=False,
dropout=0.0,
use_gpu=False,
use_bow=False,
use_kd=False,
use_posterior=False,
device=None,
unk_idx=None,
force_copy=True,
stage=None):
super().__init__()
self.src_vocab_size = src_vocab_size
self.tgt_vocab_size = tgt_vocab_size
self.cue_vocab_size = cue_vocab_size
self.goal_vocab_size = goal_vocab_size
self.embed_size = embed_size
self.hidden_size = hidden_size
self.padding_idx = padding_idx
self.num_layers = num_layers
self.bidirectional = bidirectional
self.attn_mode = attn_mode
self.with_bridge = with_bridge
self.tie_embedding = tie_embedding
self.dropout = dropout
self.use_gpu = use_gpu
self.use_bow = use_bow
self.use_kd = use_kd
self.use_posterior = use_posterior
self.baseline = 0
self.device = device if device >= 0 else "cpu"
self.unk_idx = unk_idx
self.force_copy = force_copy
self.stage = stage
# the utterance embedding
enc_embedder = Embedder(num_embeddings=self.src_vocab_size,
embedding_dim=self.embed_size,
padding_idx=self.padding_idx)
self.utt_encoder = RNNEncoder(input_size=self.embed_size,
hidden_size=self.hidden_size,
embedder=enc_embedder,
num_layers=self.num_layers,
bidirectional=self.bidirectional,
dropout=self.dropout)
if self.with_bridge:
self.utt_bridge = nn.Sequential(
nn.Linear(self.hidden_size, self.hidden_size), nn.Tanh())
self.goal_bridge = nn.Sequential(
nn.Linear(self.hidden_size, self.hidden_size), nn.Tanh())
# self.prior_query_mlp = nn.Sequential(nn.Linear(self.hidden_size * 2, self.hidden_size), nn.Tanh())
self.fc1 = nn.Linear(self.hidden_size, self.hidden_size)
self.fc2 = nn.Linear(self.hidden_size, self.hidden_size)
self.fc3 = nn.Linear(self.hidden_size * 2, 1)
if self.tie_embedding:
# share the same embedding with utt encoder
assert self.src_vocab_size == self.tgt_vocab_size == self.cue_vocab_size == self.goal_vocab_size
self.dec_embedder = enc_embedder
knowledge_embedder = enc_embedder
goal_embedder = enc_embedder
else:
self.dec_embedder = Embedder(num_embeddings=self.tgt_vocab_size,
embedding_dim=self.embed_size,
padding_idx=self.padding_idx)
knowledge_embedder = Embedder(num_embeddings=self.cue_vocab_size,
embedding_dim=self.embed_size,
padding_idx=self.padding_idx)
goal_embedder = Embedder(num_embeddings=self.goal_vocab_size,
embedding_dim=self.embed_size,
padding_idx=self.padding_idx)
self.knowledge_encoder = RNNEncoder(input_size=self.embed_size,
hidden_size=self.hidden_size,
embedder=knowledge_embedder,
num_layers=self.num_layers,
bidirectional=self.bidirectional,
dropout=self.dropout)
self.goal_encoder = RNNEncoder(input_size=self.embed_size,
hidden_size=self.hidden_size,
embedder=goal_embedder,
num_layers=self.num_layers,
bidirectional=self.bidirectional,
dropout=self.dropout)
self.prior_attention = Attention(query_size=self.hidden_size,
memory_size=self.hidden_size,
hidden_size=self.hidden_size,
mode="dot",
device=self.device)
self.posterior_attention = Attention(query_size=self.hidden_size,
memory_size=self.hidden_size,
hidden_size=self.hidden_size,
mode="dot",
device=self.device)
self.decoder = Decoder(input_size=self.embed_size,
hidden_size=self.hidden_size,
output_size=self.tgt_vocab_size,
embedder=self.dec_embedder,
num_layers=self.num_layers,
attn_mode=self.attn_mode,
memory_size=self.hidden_size,
dropout=self.dropout,
device=self.device)
self.softmax = nn.Softmax(dim=-1)
self.sigmoid = nn.Sigmoid()
self.tanh = nn.Tanh()
if self.use_bow:
self.bow_output_layer = nn.Sequential(
nn.Linear(in_features=self.hidden_size,
out_features=self.hidden_size), nn.Tanh(),
nn.Linear(in_features=self.hidden_size,
out_features=self.tgt_vocab_size),
nn.LogSoftmax(dim=-1))
if self.use_kd:
self.knowledge_dropout = nn.Dropout(self.dropout)
if self.padding_idx is not None:
self.weight = torch.ones(self.tgt_vocab_size)
self.weight[self.padding_idx] = 0
else:
self.weight = None
self.nll_loss = NLLLoss(weight=self.weight,
ignore_index=self.padding_idx,
reduction='mean')
self.copy_gen_loss = CopyGeneratorLoss(vocab_size=self.tgt_vocab_size,
force_copy=self.force_copy,
unk_index=self.unk_idx,
ignore_index=self.padding_idx)
self.kl_loss = torch.nn.KLDivLoss(reduction="mean")
if self.use_gpu:
self.cuda()
self.weight = self.weight.cuda()
def load_decoder(path):
print('Loading decoder')
decoder = Decoder()
decoder.load_state_dict(torch.load(args.decoder))
return decoder
def __init__(self, encoder_weights_path):
self.encoder = Encoder(encoder_weights_path)
self.decoder = Decoder()
def __init__(self):
self.encoder = Encoder()
self.decoder = Decoder()
def find_leaders(self):
logger.debug('Finding leaders...')
# A leader is a mark that identifies either the start or end of a basic block
# address is the positional offset of the leader within the instruction bytes
# type can be either S (starting) or E (ending)
Leader = collections.namedtuple('leader', ['address', 'type'])
# Set to contain all the leaders. We use a set to prevent duplicates
leader_set = set()
# The entrypoint is automatically the start of a basic block, and hence a start leader
leader_set.add(Leader(self.entrypoint, 'S'))
logger.debug('Start leader at {}'.format(self.entrypoint))
# Queue to contain list of addresses, from where linear sweep disassembling would start
analysis_Q = Queue.Queue()
# Start analysis from the entrypoint
analysis_Q.put(self.entrypoint)
# Already analyzed addresses must not be analyzed later, else we would get into an infinite loop
# while processing instructions that branch backwards to an previously analyzed address.
# The already_analyzed set would contains the addresses that have been previously encountered.
already_analyzed = set()
# Create the decoder
dec = Decoder(self.insBytes)
while not analysis_Q.empty():
addr = analysis_Q.get()
while True:
ins = dec.decode_at(addr)
# Put the current address into the already_analyzed set
already_analyzed.add(addr)
# If current instruction is a return, stop disassembling further.
# current address is an end leader
if ins.is_ret():
leader_set.add(Leader(addr, 'E'))
logger.debug('End leader at {}'.format(addr))
break
# If current instruction is control flow, stop disassembling further.
# the current instr is an end leader, control flow target(s) is(are) start leaders
if ins.is_control_flow():
# Current instruction is an end leader
leader_set.add(Leader(addr, 'E'))
logger.debug('End leader at {}'.format(addr))
# The list of addresses where execution is expected to transfer are starting leaders
for target in self.get_next_ins_addresses(ins,
addr).values():
leader_set.add(Leader(target, 'S'))
logger.debug('Start leader at {}'.format(addr))
# Put into analysis queue if not already analyzed
if target not in already_analyzed:
analysis_Q.put(target)
break
# Current instruction is not control flow
else:
# Get cross refs
xref = self.get_ins_xref(ins, addr)
nextAddress = self.get_next_ins_addresses(ins,
addr).values()
# Non control flow instruction should only have a single possible next address
assert len(nextAddress) == 1
# The immediate next instruction positionally
addr = nextAddress[0]
# If the instruction has xrefs, they are start leaders
if xref is not None:
leader_set.add(Leader(xref, 'S'))
logger.debug('Start leader at {}'.format(xref))
# Put into analysis queue if not already analyzed
if xref not in already_analyzed:
analysis_Q.put(xref)
# Comparator function to sort the leaders according to increasing offsets
def __leaderSortFunc(elem1, elem2):
if elem1.address != elem2.address:
return elem1.address - elem2.address
else:
if elem1.type == 'S':
return -1
else:
return 1
logger.debug('Found {} leaders'.format(len(leader_set)))
self.leaders = sorted(leader_set, cmp=__leaderSortFunc)
def construct_basic_blocks(self):
"""
Once we have obtained the leaders, i.e. the boundaries where a basic block may start or end,
we need to build the basic blocks by parsing the leaders. A basic block spans from the starting leader
upto the immediate next end leader as per their addresses.
"""
logger.debug('Constructing basic blocks...')
idx = 0
dec = Decoder(self.insBytes)
while idx < len(self.leaders):
# Get a pair of leaders
leader1, leader2 = self.leaders[idx], self.leaders[idx + 1]
# Get the addresses of the respective leaders
addr1, addr2 = leader1.address, leader2.address
# Create a new basic block
bb = BasicBlock()
# Set the address of the basic block
bb.address = addr1
# The offset variable is used track the position of the individual instructions within the basic block
offset = 0
# Store the basic block at the entrypoint separately
if addr1 == self.entrypoint:
self.bb_graph.add_node(bb, isEntry=True)
else:
self.bb_graph.add_node(bb)
# Add the basic block to the graph
self.bb_graph.add_node(bb)
# Leader1 is start leader, leader2 is end leader
# All instructions inclusive of leader1 and leader2 are part of this basic block
if leader1.type == 'S' and leader2.type == 'E':
logger.debug(
'Creating basic block {} spanning from {} to {}, both inclusive'
.format(hex(id(bb)), leader1.address, leader2.address))
while addr1 + offset <= addr2:
ins = dec.decode_at(addr1 + offset)
bb.add_instruction(ins)
offset += ins.size
idx += 2
# Both Leader1 and leader2 are start leader
# Instructions inclusive of leader1 but exclusive of leader2 are part of this basic block
elif leader1.type == 'S' and leader2.type == 'S':
logger.debug(
'Creating basic block {} spanning from {} to {}, end exclusive'
.format(hex(id(bb)), leader1.address, leader2.address))
while addr1 + offset < addr2:
ins = dec.decode_at(addr1 + offset)
bb.add_instruction(ins)
offset += ins.size
idx += 1
logger.debug('{} basic blocks created'.format(
self.bb_graph.number_of_nodes()))
def process_current_file_test():
dec = Decoder(config)
dec.process_current_file()
def __init__(self, model_pre_path):
print("------------------------------------")
print(model_pre_path)
self.encoder = Encoder(model_pre_path)
self.decoder = Decoder(model_pre_path)
plt.axis('off')
plt.tight_layout()
plt.show()
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Show, Attend and Tell Caption Generator')
parser.add_argument('--img-path', type=str, help='path to image')
parser.add_argument('--network',
choices=['vgg19', 'resnet152'],
default='vgg19',
help='Network to use in the encoder (default: vgg19)')
parser.add_argument('--run_dir', type=str, help='results/Ient')
parser.add_argument('--data-path',
type=str,
default='../../datasets/MSCOCO',
help='path to data (default: data/coco)')
args = parser.parse_args()
word_dict = json.load(open(args.data_path + '/word_dict.json', 'r'))
vocabulary_size = len(word_dict)
encoder = Encoder(network=args.network)
decoder = Decoder(vocabulary_size, encoder.dim)
trainer = Trainer(encoder, decoder, None, None)
saver = Saver(trainer, args.run_dir)
encoder.eval()
decoder.eval()
generate_caption_visualization(encoder, decoder, args.img_path, word_dict)
def main(args):
# Construct Solver
# data
tr_dataset = AudioDataset(args.train_json,
args.batch_size,
args.maxlen_in,
args.maxlen_out,
batch_frames=args.batch_frames)
cv_dataset = AudioDataset(args.valid_json,
args.batch_size,
args.maxlen_in,
args.maxlen_out,
batch_frames=args.batch_frames)
tr_loader = AudioDataLoader(tr_dataset,
batch_size=1,
num_workers=args.num_workers,
shuffle=args.shuffle,
LFR_m=args.LFR_m,
LFR_n=args.LFR_n)
cv_loader = AudioDataLoader(cv_dataset,
batch_size=1,
num_workers=args.num_workers,
LFR_m=args.LFR_m,
LFR_n=args.LFR_n)
# load dictionary and generate char_list, sos_id, eos_id
char_list, sos_id, eos_id = process_dict(args.dict)
vocab_size = len(char_list)
data = {'tr_loader': tr_loader, 'cv_loader': cv_loader}
# model
encoder = Encoder(args.d_input * args.LFR_m,
args.n_layers_enc,
args.n_head,
args.d_k,
args.d_v,
args.d_model,
args.d_inner,
dropout=args.dropout,
pe_maxlen=args.pe_maxlen)
decoder = Decoder(
sos_id,
eos_id,
vocab_size,
args.d_word_vec,
args.n_layers_dec,
args.n_head,
args.d_k,
args.d_v,
args.d_model,
args.d_inner,
dropout=args.dropout,
tgt_emb_prj_weight_sharing=args.tgt_emb_prj_weight_sharing,
pe_maxlen=args.pe_maxlen)
model = Transformer(encoder, decoder)
print(model)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
#model.cuda()
model.to(device)
# optimizer
optimizier = TransformerOptimizer(
torch.optim.Adam(model.parameters(), betas=(0.9, 0.98), eps=1e-09),
args.k, args.d_model, args.warmup_steps)
# solver
solver = Solver(data, model, optimizier, args)
solver.train()
for start in range(0, len(indices), batch_size):
batch_indices = indices_shufled[start:start + batch_size]
emb_batch = np.ndarray((batch_size, EMB_SIZE))
img_batch = np.ndarray((len(batch_indices), 160 * 160 * 3))
for i, img_id in enumerate(batch_indices):
img_path = 'img_align_celeba_160/{0:06d}.jpg'.format(img_id + 1)
img_batch[i] = np.array(imageio.imread(img_path) / 255).reshape(
1, -1)
emb_batch[i] = embeddings[img_id]
yield img_batch, emb_batch
decoder = Decoder(in_features=EMB_SIZE, out_features=160 * 160 * 3)
embeddings = np.ndarray(shape=(EMB_COUNT, EMB_SIZE))
with open('embeddings_float.txt') as f:
for i, line in enumerate(f):
embeddings[i, :] = np.array(
list(map(np.float64,
line.strip()[4:].split())))
train_indices = np.random.choice(range(0, EMB_COUNT),
size=int(EMB_COUNT * 0.9),
replace=False)
train_indices.sort()
test_indices = np.array(list(set(range(0, EMB_COUNT)) - set(train_indices)))
test_indices.sort()
vocab = pickle.load(f)
# load data
train_loader = get_loader('train', vocab, batch_size)
#############
# Init model
#############
criterion = nn.CrossEntropyLoss().to(device)
else:
encoder = Encoder().to(device)
encoder_optimizer = torch.optim.Adam(params=encoder.parameters(),
lr=1e-4)
decoder = Decoder(vocab_size=len(vocab),use_glove=glove_model, use_bert=bert_model, vocab=vocab, device=device, BertTokenizer=tokenizer, BertModel=BertModel).to(device)
decoder_optimizer = torch.optim.Adam(params=decoder.parameters(),lr=decoder_lr)
###############
# Train model
###############
def train():
print("Create directory for checkpoints")
if not os.path.exists(config.model_dir):
os.mkdir(config.model_dir)
print("Started training...")
for epoch in tqdm(range(num_epochs)):
def train(exp=None):
"""
main function to run the training
"""
encoder = Encoder(encoder_params[0], encoder_params[1]).cuda()
decoder = Decoder(decoder_params[0], decoder_params[1]).cuda()
net = ED(encoder, decoder)
run_dir = "./runs/" + TIMESTAMP
if not os.path.isdir(run_dir):
os.makedirs(run_dir)
# tb = SummaryWriter(run_dir)
# initialize the early_stopping object
early_stopping = EarlyStopping(patience=20, verbose=True)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if torch.cuda.device_count() > 1:
net = nn.DataParallel(net)
net.to(device)
if os.path.exists(args.checkpoint) and args.continue_train:
# load existing model
print("==> loading existing model")
model_info = torch.load(args.checkpoint)
net.load_state_dict(model_info["state_dict"])
optimizer = torch.optim.Adam(net.parameters())
optimizer.load_state_dict(model_info["optimizer"])
cur_epoch = model_info["epoch"] + 1
else:
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
cur_epoch = 0
lossfunction = nn.MSELoss().cuda()
optimizer = optim.Adam(net.parameters(), lr=args.lr)
pla_lr_scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
factor=0.5,
patience=4,
verbose=True)
# to track the average training loss per epoch as the model trains
avg_train_losses = []
# to track the average validation loss per epoch as the model trains
avg_valid_losses = []
# pnsr ssim
avg_psnrs = {}
avg_ssims = {}
for j in range(args.frames_output):
avg_psnrs[j] = []
avg_ssims[j] = []
if args.checkdata:
# Checking dataloader
print("Checking Dataloader!")
t = tqdm(trainLoader, leave=False, total=len(trainLoader))
for i, (idx, targetVar, inputVar, _, _) in enumerate(t):
assert targetVar.shape == torch.Size([
args.batchsize, args.frames_output, 1, args.data_h, args.data_w
])
assert inputVar.shape == torch.Size([
args.batchsize, args.frames_input, 1, args.data_h, args.data_w
])
print("TrainLoader checking is complete!")
t = tqdm(validLoader, leave=False, total=len(validLoader))
for i, (idx, targetVar, inputVar, _, _) in enumerate(t):
assert targetVar.shape == torch.Size([
args.batchsize, args.frames_output, 1, args.data_h, args.data_w
])
assert inputVar.shape == torch.Size([
args.batchsize, args.frames_input, 1, args.data_h, args.data_w
])
print("ValidLoader checking is complete!")
# mini_val_loss = np.inf
for epoch in range(cur_epoch, args.epochs + 1):
# to track the training loss as the model trains
train_losses = []
# to track the validation loss as the model trains
valid_losses = []
psnr_dict = {}
ssim_dict = {}
for j in range(args.frames_output):
psnr_dict[j] = 0
ssim_dict[j] = 0
image_log = []
if exp is not None:
exp.log_metric("epoch", epoch)
###################
# train the model #
###################
t = tqdm(trainLoader, leave=False, total=len(trainLoader))
for i, (idx, targetVar, inputVar, _, _) in enumerate(t):
inputs = inputVar.to(device) # B,S,C,H,W
label = targetVar.to(device) # B,S,C,H,W
optimizer.zero_grad()
net.train()
pred = net(inputs) # B,S,C,H,W
loss = lossfunction(pred, label)
loss_aver = loss.item() / args.batchsize
train_losses.append(loss_aver)
loss.backward()
torch.nn.utils.clip_grad_value_(net.parameters(), clip_value=10.0)
optimizer.step()
t.set_postfix({
"trainloss": "{:.6f}".format(loss_aver),
"epoch": "{:02d}".format(epoch),
})
# tb.add_scalar('TrainLoss', loss_aver, epoch)
######################
# validate the model #
######################
with torch.no_grad():
net.eval()
t = tqdm(validLoader, leave=False, total=len(validLoader))
for i, (idx, targetVar, inputVar, _, _) in enumerate(t):
inputs = inputVar.to(device)
label = targetVar.to(device)
pred = net(inputs)
loss = lossfunction(pred, label)
loss_aver = loss.item() / args.batchsize
# record validation loss
valid_losses.append(loss_aver)
for j in range(args.frames_output):
psnr_dict[j] += psnr(pred[:, j], label[:, j])
ssim_dict[j] += ssim(pred[:, j], label[:, j])
# print ("validloss: {:.6f}, epoch : {:02d}".format(loss_aver,epoch),end = '\r', flush=True)
t.set_postfix({
"validloss": "{:.6f}".format(loss_aver),
"epoch": "{:02d}".format(epoch),
})
if i % 500 == 499:
for k in range(args.frames_output):
image_log.append(label[0, k].unsqueeze(0).repeat(
1, 3, 1, 1))
image_log.append(pred[0, k].unsqueeze(0).repeat(
1, 3, 1, 1))
upload_images(
image_log,
epoch,
exp=exp,
im_per_row=2,
rows_per_log=int(len(image_log) / 2),
)
# tb.add_scalar('ValidLoss', loss_aver, epoch)
torch.cuda.empty_cache()
# print training/validation statistics
# calculate average loss over an epoch
train_loss = np.average(train_losses)
valid_loss = np.average(valid_losses)
avg_train_losses.append(train_loss)
avg_valid_losses.append(valid_loss)
for j in range(args.frames_output):
avg_psnrs[j].append(psnr_dict[j] / i)
avg_ssims[j].append(ssim_dict[j] / i)
epoch_len = len(str(args.epochs))
print_msg = (f"[{epoch:>{epoch_len}}/{args.epochs:>{epoch_len}}] " +
f"train_loss: {train_loss:.6f} " +
f"valid_loss: {valid_loss:.6f}" +
f"PSNR_1: {psnr_dict[0] / i:.6f}" +
f"SSIM_1: {ssim_dict[0] / i:.6f}")
# print(print_msg)
# clear lists to track next epoch
if exp is not None:
exp.log_metric("TrainLoss", train_loss)
exp.log_metric("ValidLoss", valid_loss)
exp.log_metric("PSNR_1", psnr_dict[0] / i)
exp.log_metric("SSIM_1", ssim_dict[0] / i)
pla_lr_scheduler.step(valid_loss) # lr_scheduler
model_dict = {
"epoch": epoch,
"state_dict": net.state_dict(),
"optimizer": optimizer.state_dict(),
"avg_psnrs": avg_psnrs,
"avg_ssims": avg_ssims,
"avg_valid_losses": avg_valid_losses,
"avg_train_losses": avg_train_losses,
}
save_flag = False
if epoch % args.save_every == 0:
torch.save(
model_dict,
save_dir + "/" +
"checkpoint_{}_{:.6f}.pth".format(epoch, valid_loss.item()),
)
print("Saved" +
"checkpoint_{}_{:.6f}.pth".format(epoch, valid_loss.item()))
save_flag = True
if avg_psnrs[0][-1] == max(avg_psnrs[0]) and not save_flag:
torch.save(
model_dict,
save_dir + "/" + "bestpsnr_1.pth",
)
print("Best psnr found and saved")
save_flag = True
if avg_ssims[0][-1] == max(avg_ssims[0]) and not save_flag:
torch.save(
model_dict,
save_dir + "/" + "bestssim_1.pth",
)
print("Best ssim found and saved")
save_flag = True
if avg_valid_losses[-1] == min(avg_valid_losses) and not save_flag:
torch.save(
model_dict,
save_dir + "/" + "bestvalidloss.pth",
)
print("Best validloss found and saved")
save_flag = True
if not save_flag:
torch.save(
model_dict,
save_dir + "/" + "checkpoint.pth",
)
print("The latest normal checkpoint saved")
early_stopping(valid_loss.item(), model_dict, epoch, save_dir)
if early_stopping.early_stop:
print("Early stopping")
break
with open("avg_train_losses.txt", "wt") as f:
for i in avg_train_losses:
print(i, file=f)
with open("avg_valid_losses.txt", "wt") as f:
for i in avg_valid_losses:
print(i, file=f)
def main():
use_cuda = args.use_cuda
print("Cuda set to {} | Cuda availability: {}".format(
use_cuda, torch.cuda.is_available()))
experiment = "vae_latent3"
#logger = SummaryWriter(log_dir='./logs', comment=experiment)
train_data = UnlabeledContact(
data='/home/ygx/data/fspeptide/fs_peptide.npy')
print('Number of samples: {}'.format(len(train_data)))
trainloader = DataLoader(train_data, batch_size=args.batch_size)
# Contact matrices are 21x21
input_size = 441
encoder = Encoder(input_size=input_size, latent_size=3)
decoder = Decoder(latent_size=3, output_size=input_size)
vae = VAE(encoder, decoder, use_cuda=use_cuda)
#criterion = nn.BCELoss()
if use_cuda:
encoder = encoder.cuda().half()
decoder = decoder.cuda().half()
vae = vae.cuda().half()
#criterion = criterion.cuda().half()
optimizer = optim.SGD(vae.parameters(), lr=0.01)
losses = AverageMeter()
epoch_loss = 0
total_loss = 0
for epoch in range(100):
for batch_idx, data in enumerate(trainloader):
inputs = data['cont_matrix']
inputs = inputs.resize_(args.batch_size, 1, 21, 21)
inputs = inputs.float()
if use_cuda:
inputs = inputs.cuda().half()
inputs = Variable(inputs)
# Compute output
optimizer.zero_grad()
dec = vae(inputs)
# Measure the loss
#kl = kl_loss(vae.z_mean, vae.z_sigma)
#loss = criterion(dec, inputs) #+ kl # Adding KL is caussing loss > 1
loss = loss_function(dec, inputs, vae.z_mean, vae.z_sigma)
losses.update(loss.data[0], inputs.size(0))
# Compute the gradient
loss.backward()
optimizer.step()
epoch_loss += loss.data[0]
# Logging
# Adding graph is a lot of overhead
#logger.add_graph_onnx(vae)
# log loss values every iteration
#logger.add_scalar('data/(train)loss_val', losses.val, batch_idx + 1)
#logger.add_scalar('data/(train)loss_avg', losses.avg, batch_idx + 1)
# log the layers and layers gradient histogram and distributions
#for tag, value in vae.named_parameters():
# tag = tag.replace('.', '/')
# logger.add_histogram('model/(train)' + tag, to_numpy(value), batch_idx + 1)
# logger.add_histogram('model/(train)' + tag + '/grad', to_numpy(value.grad), batch_idx + 1)
# log the outputs of the autoencoder
#logger.add_image('model/(train)output', make_grid(dec.data), batch_idx + 1)
if batch_idx % args.log_interval == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(trainloader.dataset),
100. * batch_idx / len(trainloader), loss.data[0]))
#if epoch < 10:
# Get latent encoding
#latent_array = encoder(inputs).data[0].cpu().numpy()
#filename = 'latent_epoch' + str(epoch)
#np.save('./latent_saves/kl_bce_latent3/' + filename, latent_array)
# Get reconstructed image
#reconstructed_array = vae(inputs).data[0].cpu().numpy().reshape(21, 21)
#recon_filename = 'reconstructed_epoch' + str(epoch)
#np.save('./reconstruct_saves/kl_bce_latent3/' + recon_filename, reconstructed_array)
if epoch % 10 == 0:
torch.save(vae.state_dict(), args.save_path + 'epoch' + str(epoch))
#latent_array = encoder(inputs).data[0].cpu().numpy()
#filename = 'latent_epoch' + str(epoch)
#np.save('./latent_saves/kl_bce_latent3/' + filename, latent_array)
reconstructed_array = vae(
inputs).data[0].cpu().float().numpy().reshape(21, 21)
recon_filename = 'reconstructed_epoch' + str(epoch)
np.save('./reconstruct_saves/kl_bce_latent3/' + recon_filename,
reconstructed_array)
def __init__(self, loop):
self.loop = loop
self.decoder = Decoder()
self.msg_get_anchorlist = b'##\x06\x00A\x00'
self.msg_get_taglist = b'##\x06\x00T\x00'
train_data = utils.dataset(train_path, batch_size * train_steps_each_iter)
dev_data = utils.dataset(dev_path, batch_size * dev_steps_each_iter)
train_iter = train_data.make_one_shot_iterator()
dev_iter = dev_data.make_one_shot_iterator()
train_saveable = tf.data.experimental.make_saveable_from_iterator(
train_iter)
dev_saveable = tf.data.experimental.make_saveable_from_iterator(dev_iter)
saveable = [train_saveable, dev_saveable]
# model
with open("model/hparams.json") as f:
config = json.load(f)
with tpu_graph.as_default():
encoder = Encoder(config, "encoder")
decoder = Decoder(config, "decoder")
encoded_length = 1
if encoder.mode == "attention":
encoded_length = encoder.aggregation.query_num
# optimizer
with tpu_graph.as_default():
lr = tf.Variable(1e-4, name="lr")
optimizer = tf.train.AdamOptimizer(lr)
optimizer = tf.tpu.CrossShardOptimizer(optimizer)
def get_weights():
weights = []
if phase == 1:
if encoder.mode == "attention":
def main():
train_data = SentenceDataset(args.train_file,
encoding_type=args.encoding_type,
filter_threshold=args.filter_threshold)
val_data = SentenceDataset(args.val_file,
encoding_type=args.encoding_type,
filter_threshold=args.filter_threshold)
train_loader = torch.utils.data.DataLoader(train_data,
args.batch_size,
shuffle=True)
val_loader = torch.utils.data.DataLoader(val_data, args.batch_size)
print(len(train_loader))
input_dim = len(train_data.vocab.source_vocab)
output_dim = len(train_data.vocab.target_vocab)
static = args.embedding_type == 'static'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
enc_embedding = Embeddings(input_dim, args.hidden_dim, args.max_len,
device, static)
encoder_layer = EncoderLayer(args.hidden_dim, args.num_enc_heads,
args.inner_dim, args.dropout)
encoder = Encoder(enc_embedding, encoder_layer, args.num_enc_layers,
args.dropout)
dec_embedding = Embeddings(input_dim, args.hidden_dim, args.max_len,
device, static)
decoder_layer = DecoderLayer(args.hidden_dim, args.num_dec_heads,
args.inner_dim, args.dropout)
decoder = Decoder(output_dim, args.hidden_dim, dec_embedding,
decoder_layer, args.num_dec_layers, args.dropout)
pad_id = train_data.vocab.source_vocab['<pad>']
model = Transformer(encoder, decoder, pad_id, device)
print('Transformer has {:,} trainable parameters'.format(
count_parames(model)))
if args.load_model is not None:
model.load(args.load_model)
else:
model.apply(init_weights)
if args.mode == 'test':
inferencer = Inferencer(model, train_data.vocab, device)
greedy_out = inferencer.infer_greedy(
'helo world, I m testin a typo corector')
print(greedy_out)
elif args.mode == 'train':
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
loss_function = nn.NLLLoss(ignore_index=pad_id)
print('Started training...')
train(model, train_loader, val_loader, optimizer, loss_function,
device)
else:
raise ValueError('Mode not recognized')
def __init__(self, batchloader, is_training, ru):
self.batchloader = batchloader
self.ru = ru
self.is_training = is_training
self.lr = tf.placeholder(tf.float32, shape=(), name="learning_rate")
with tf.name_scope("Placeholders"):
self.encoder_input = tf.placeholder(tf.int64,
shape=(FLAGS.BATCH_SIZE, FLAGS.SEQ_LEN),
name="encoder_input")
self.decoder_input = tf.placeholder(tf.int64,
shape=(FLAGS.BATCH_SIZE, FLAGS.SEQ_LEN),
name="decoder_input")
self.target = tf.placeholder(tf.int64,
shape=(FLAGS.BATCH_SIZE, FLAGS.SEQ_LEN),
name="target")
encoder_input_t = tf.transpose(self.encoder_input, perm=[1, 0])
self.encoder_input_list = [] # debug
decoder_input_t = tf.transpose(self.decoder_input, perm=[1, 0])
self.decoder_input_list = []
target_t = tf.transpose(self.target, perm=[1, 0])
self.target_list = []
self.step = tf.placeholder(tf.float32, shape=(), name="step")
for i in range(FLAGS.SEQ_LEN):
self.encoder_input_list.append(encoder_input_t[i])
assert self.encoder_input_list[i].shape == (FLAGS.BATCH_SIZE)
self.decoder_input_list.append(decoder_input_t[i])
assert self.decoder_input_list[i].shape == (FLAGS.BATCH_SIZE)
self.target_list.append(target_t[i])
assert self.target_list[i].shape == (FLAGS.BATCH_SIZE)
with tf.variable_scope("Embedding"):
self.embedding = tf.get_variable(name="embedding",
shape=[FLAGS.VOCAB_SIZE, FLAGS.EMBED_SIZE],
dtype=tf.float32,
initializer=tf.random_normal_initializer(stddev=0.1))
with tf.variable_scope("Encoder"):
self.encoder_input_embedded = tf.nn.embedding_lookup(self.embedding, self.encoder_input)
self.encoder = Encoder(self.encoder_input_embedded,
is_training = self.is_training,
ru = self.ru)
with tf.name_scope("Latent_variables"):
if self.is_training:
self.latent_variables = self.encoder.latent_variables
else:
self.latent_variables = tf.placeholder(tf.float32,
shape=(FLAGS.BATCH_SIZE,
FLAGS.LATENT_VARIABLE_SIZE),
name="latent_variables_input")
with tf.variable_scope("Decoder"):
self.decoder = Decoder(self.decoder_input_list,
self.latent_variables,
self.embedding,
self.batchloader,
is_training = self.is_training,
ru = self.ru)
with tf.name_scope("Loss"):
self.aux_logits = self.decoder.aux_logits
self.rnn_logits = self.decoder.rnn_logits
self.kld = tf.reduce_mean(-0.5 *
tf.reduce_sum(self.encoder.logvar
- tf.square(self.encoder.mu)
- tf.exp(self.encoder.logvar)
+ 1,
axis=1))
self.kld_weight = tf.clip_by_value((self.step - FLAGS.KLD_ANNEAL_START) /
(FLAGS.KLD_ANNEAL_END - FLAGS.KLD_ANNEAL_START),
0, 1)
aux_losses = [tf.nn.sparse_softmax_cross_entropy_with_logits( \
logits=logits, labels=targets) \
for logits, targets in zip(self.aux_logits, self.target_list)]
self.aux_loss = tf.reduce_mean(aux_losses) * FLAGS.SEQ_LEN
rnn_losses = [tf.nn.sparse_softmax_cross_entropy_with_logits( \
logits=logits, labels=targets) \
for logits, targets in zip(self.rnn_logits, self.target_list)]
self.rnn_loss = tf.reduce_mean(rnn_losses) * FLAGS.SEQ_LEN
self.loss = self.rnn_loss + FLAGS.ALPHA * self.aux_loss + \
self.kld_weight * self.kld
with tf.name_scope("Summary"):
if is_training:
loss_summary = tf.summary.scalar("loss", self.loss, family="train_loss")
rnn_loss_summary = tf.summary.scalar("rnn_loss", self.rnn_loss, family="train_loss")
aux_loss_summary = tf.summary.scalar("aux_loss", self.aux_loss, family="train_loss")
kld_summary = tf.summary.scalar("kld", self.kld, family="kld")
kld_weight_summary = tf.summary.scalar("kld_weight", self.kld_weight, family="parameters")
mu_summary = tf.summary.histogram("mu", tf.reduce_mean(self.encoder.mu, 0))
var_summary = tf.summary.histogram("var", tf.reduce_mean(tf.exp(self.encoder.logvar), 0))
lr_summary = tf.summary.scalar("lr", self.lr, family="parameters")
self.merged_summary = tf.summary.merge([loss_summary, rnn_loss_summary,
aux_loss_summary, kld_summary,
kld_weight_summary, mu_summary, var_summary,
lr_summary])
else:
valid_rnn_loss_summary = tf.summary.scalar("valid_rnn_loss", self.rnn_loss, family="valid_loss")
valid_aux_loss_summary = tf.summary.scalar("valid_aux_loss", self.aux_loss, family="valid_loss")
self.merged_summary = tf.summary.merge([valid_rnn_loss_summary,
valid_aux_loss_summary])
tvars = tf.trainable_variables()
if(self.is_training):
with tf.name_scope("Optimizer"):
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars),
FLAGS.MAX_GRAD)
optimizer = tf.train.AdamOptimizer(self.lr)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
def __init__(self, logger):
Logger.log.debug('{} initializing....'.format(__name__))
self.logger = logger
self.config = Config(logger=self.logger)
self.support = Support(config=self.config, logger=self.logger)
self.gpio = Gpio(config=self.config, logger=self.logger)
self.pollperm = Pollperm(logger=self.logger)
self.decoder = Decoder(config=self.config,
logger=self.logger,
gpio=self.gpio)
self.spi = SPI(config=self.config,
logger=self.logger,
decoder=self.decoder,
pollperm=self.pollperm)
self.codegen = Codegen(config=self.config,
logger=self.logger,
gpio=self.gpio,
spi=self.spi)
self.securitylevel = SecurityLevel(logger=self.logger)
self.gui = Mainwindow(self,
codegen=self.codegen,
config=self.config,
logger=self.logger,
support=self.support,
securitylevel=self.securitylevel)
self.switches = Switches(config=self.config,
logger=self.logger,
spi=self.spi,
gui=self.gui)
self.currentsense = CurrentSense(logger=self.logger,
spi=self.spi,
decoder=self.decoder,
gui=self.gui,
config=self.config)
self.pollvalues = Pollvalues(pollperm=self.pollperm,
logger=logger,
config=self.config,
currentsense=self.currentsense,
switches=self.switches,
sense_callback=self.poll_sense_callback,
switch_callback=self.poll_switch_callback)
self.securitywindow = SecurityWindow(logger=self.logger,
securitylevel=self.securitylevel)
self.window = self.gui.window
self.log = self.logger.log
self.knob_values = 0
self.rotary_0_pins = None
self.rotary_1_pins = None
self.rotary_2_pins = None
self.rotary_3_pins = None
self.rotary_0_pin_0_debounce = None
self.rotary_0_pin_1_debounce = None
self.rotary_1_pin_0_debounce = None
self.rotary_1_pin_1_debounce = None
self.rotary_2_pin_0_debounce = None
self.rotary_2_pin_1_debounce = None
self.rotary_3_pin_0_debounce = None
self.rotary_3_pin_1_debounce = None
self.gain0_val = 0
self.gain1_val = 0
self.gain_0_name = None
self.gain_1_name = None
self.gain_0_spi_channel = None
self.gain_1_spi_channel = None
self.gain_0_thresholds = None
self.gain_1_thresholds = None
self.GAIN_0_CS = None
self.GAIN_1_CS = None
self.speed0_val = 0
self.speed1_val = 0
self.speed_0_name = None
self.speed_1_name = None
self.speed_0_shape = None
self.speed_1_shape = None
self.speed_0_spi_channel = None
self.speed_1_spi_channel = None
self.speed_0_thresholds = None
self.speed_1_thresholds = None
self.screen_brightness_max = None
self.screen_brightness_min = None
self.display_brightness = None
self.spi_log_pause = False
self.SPEED_0_CS = None # 6 # SPEED SIMULATION TACH 1
self.SPEED_1_CS = None # 7 # SPEED SIMULATION TACH 2
self.load_from_config()
self.adc_scale = None
self.sense_amp_max_amps = None
self.sense_ad_vin = None # LM4128CQ1MF3.3/NOPB voltage reference
self.sense_ad_max_bits = 0 # AD7940 ADC
self.sense_scaling_factor_mv_amp = None # 110 milivolts per amp
self.sense_ad_max_scaled_value = None
self.speed0 = Speedgen(
pollperm=self.pollperm,
logger=self.logger,
config=self.config,
decoder=self.decoder,
spi=self.spi,
gpio=self.gpio,
name=self.speed_0_name,
shape=self.speed_0_shape,
spi_channel=self.speed_0_spi_channel,
chip_select=self.SPEED_0_CS,
pin_0=self.rotary_0_pins[0],
pin_1=self.rotary_0_pins[1],
pin_0_debounce=self.rotary_0_pin_0_debounce,
pin_1_debounce=self.rotary_0_pin_1_debounce,
thresholds=self.speed_0_thresholds,
callback=self.speed_callback,
commander_speed_move_callback=self.speed_move_callback)
self.speed1 = Speedgen(
pollperm=self.pollperm,
logger=self.logger,
config=self.config,
decoder=self.decoder,
spi=self.spi,
gpio=self.gpio,
name=self.speed_1_name,
shape=self.speed_1_shape,
spi_channel=self.speed_1_spi_channel,
chip_select=self.SPEED_1_CS,
pin_0=self.rotary_1_pins[0],
pin_1=self.rotary_1_pins[1],
pin_0_debounce=self.rotary_1_pin_0_debounce,
pin_1_debounce=self.rotary_1_pin_1_debounce,
thresholds=self.speed_1_thresholds,
callback=self.speed_callback,
commander_speed_move_callback=self.speed_move_callback)
self.gain0 = Gains(
pollperm=self.pollperm,
config=self.config,
logger=self.logger,
decoder=self.decoder,
spi=self.spi,
gpio=self.gpio,
name=self.gain_0_name,
spi_channel=self.gain_0_spi_channel,
chip_select=self.GAIN_0_CS,
pin_0=self.rotary_2_pins[0],
pin_1=self.rotary_2_pins[1],
pin_0_debounce=self.rotary_2_pin_0_debounce,
pin_1_debounce=self.rotary_2_pin_1_debounce,
thresholds=self.gain_0_thresholds,
callback=self.gains_callback,
commander_gain_move_callback=self.gain_move_callback)
self.gain1 = Gains(
pollperm=self.pollperm,
config=self.config,
logger=self.logger,
decoder=self.decoder,
spi=self.spi,
gpio=self.gpio,
name=self.gain_1_name,
spi_channel=self.gain_1_spi_channel,
chip_select=self.GAIN_1_CS,
pin_0=self.rotary_3_pins[0],
pin_1=self.rotary_3_pins[1],
pin_0_debounce=self.rotary_3_pin_0_debounce,
pin_1_debounce=self.rotary_3_pin_1_debounce,
thresholds=self.gain_1_thresholds,
callback=self.gains_callback,
commander_gain_move_callback=self.gain_move_callback)
self.startup_processes()
def train(self, train_file, dev_file, test_file, model_file):
self.hyperParams.show()
torch.set_num_threads(self.hyperParams.thread)
reader = Reader()
trainInsts = reader.readInstances(train_file,
self.hyperParams.maxInstance)
devInsts = reader.readInstances(dev_file, self.hyperParams.maxInstance)
testInsts = reader.readInstances(test_file,
self.hyperParams.maxInstance)
print("Training Instance: ", len(trainInsts))
print("Dev Instance: ", len(devInsts))
print("Test Instance: ", len(testInsts))
self.createAlphabet(trainInsts)
trainExamples = self.instance2Example(trainInsts)
devExamples = self.instance2Example(devInsts)
testExamples = self.instance2Example(testInsts)
self.encoder = Encoder(self.hyperParams)
self.decoder = Decoder(self.hyperParams)
indexes = []
for idx in range(len(trainExamples)):
indexes.append(idx)
encoder_parameters = filter(lambda p: p.requires_grad,
self.encoder.parameters())
encoder_optimizer = torch.optim.Adam(encoder_parameters,
lr=self.hyperParams.learningRate)
decoder_parameters = filter(lambda p: p.requires_grad,
self.decoder.parameters())
decoder_optimizer = torch.optim.Adam(decoder_parameters,
lr=self.hyperParams.learningRate)
batchBlock = len(trainExamples) // self.hyperParams.batch
for iter in range(self.hyperParams.maxIter):
print('###Iteration' + str(iter) + "###")
random.shuffle(indexes)
self.encoder.train()
self.decoder.train()
for updateIter in range(batchBlock):
exams = []
start_pos = updateIter * self.hyperParams.batch
end_pos = (updateIter + 1) * self.hyperParams.batch
for idx in range(start_pos, end_pos):
exams.append(trainExamples[indexes[idx]])
postFeats, responseFeats = self.getBatchFeatLabel(exams)
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
encoder_hidden = self.encoder.init_hidden(
self.hyperParams.batch)
encoder_output, encoder_hidden = self.encoder(
postFeats, encoder_hidden)
decoder_hidden = self.decoder.initHidden(
self.hyperParams.batch)
response_len = responseFeats.size()[1]
last_word = torch.autograd.Variable(
torch.LongTensor(1, self.hyperParams.batch))
for idx in range(self.hyperParams.batch):
last_word.data[0][idx] = self.hyperParams.responseStartID
loss = 0
for idx in range(response_len):
decoder_output, decoder_hidden = self.decoder(
encoder_hidden, decoder_hidden, last_word)
loss += torch.nn.functional.nll_loss(
decoder_output,
responseFeats.permute(1, 0)[idx])
for idy in range(self.hyperParams.batch):
last_word.data[0][idy] = self.getMaxIndex(
decoder_output[idy])
loss.backward()
print('Current: ', updateIter + 1, ", Cost:", loss.data[0])
encoder_optimizer.step()
decoder_optimizer.step()
if iter + 1 % 10 == 0:
self.encoder.eval()
self.decoder.eval()
print("Save model .....")
self.saveModel(model_file + str(iter))
print("Model model ok")
def __init__(self):
super(Generator, self).__init__()
self.encoder = Encoder()
self.decoder = Decoder()
def collect_latent_dataset():
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
data = pickle.load(open('data/pusher_dyn_relabeled.pkl',
'rb')).astype(np.float32)
data = data.transpose([1, 0, 4, 2, 3])
actions = pickle.load(open('data/pusher_actions.pkl',
'rb')).astype(np.float32)
actions = actions.transpose([1, 0, 2])
costs = pickle.load(open('data/pusher_costs.pkl', 'rb')).astype(np.float32)
costs = costs.transpose([1, 0, 2])
enc = Encoder([3, 96, 96, 8], [0, 0, 0, 8], 8).to(device)
dec = Decoder([9, 96, 96, 3], 8).to(device)
params = {}
for (k, v) in enc.named_parameters():
params['enc.' + k.replace('__', '.')] = v
for (k, v) in dec.named_parameters():
params['dec.' + k.replace('__', '.')] = v
params = init_weights(params, file='pusher_params.pkl', device=device)
from utils import threshold_latent
new_dataset = []
for i in range(data.shape[1]):
print(i)
cand_seq = []
for t in range(data.shape[0]):
ims_tensor = torch.Tensor(data[t, i].reshape(1, 3, 64, 64) /
np.max(data[t, i])).to(device)
action_tensor = torch.Tensor(actions[t, i]).to(device)
cost_tensor = torch.Tensor(costs[t, i]).to(device)
latent = 1 - torch.exp(-enc(ims_tensor))
#TODO: make general
latent = latent[0, [2, 3, 5], :, :]
if t == 0:
threshed_latent = threshold_latent(latent)
prev_action = action_tensor
prev_cost = cost_tensor
if threshed_latent is None:
cand_seq = []
prev_latent = None
prev_action = action_tensor
prev_cost = cost_tensor
continue
else:
cand_seq = [[
torch.stack(k, dim=0) if k else []
for k in threshed_latent
]]
prev_latent = latent
prev_action = action_tensor
prev_cost = cost_tensor
else:
threshed_latent = threshold_latent(latent, prev_latent)
if threshed_latent is None:
threshed_latent = threshold_latent(latent)
if threshed_latent is None:
cand_seq = []
prev_latent = None
prev_action = action_tensor
prev_cost = cost_tensor
continue
else:
cand_seq = [[
torch.stack(k, dim=0) if k else []
for k in threshed_latent
]]
prev_latent = latent
prev_action = action_tensor
prev_cost = cost_tensor
else:
cand_seq.append([
torch.stack(k, dim=0) if k else []
for k in threshed_latent
])
if len(cand_seq) == 3:
new_dataset.append(
copy.deepcopy(cand_seq) +
[prev_action.clone(),
prev_cost.clone()])
cand_seq.pop(0)
prev_latent = latent
prev_action = action_tensor
prev_cost = cost_tensor
pickle.dump(new_dataset, open('data/pusher_dyn_latent.pkl', 'wb'))
def process_file_test():
dec = Decoder(config)
processed_data = dec.process_file(
Path(config['paths']['experiment_data_path']))
for key, value in processed_data.items():
print(value)
