def setUp(self):
"""sets up objects for testing later
"""
self.test_model1 = BaseModel()
self.test_model2 = BaseModel()
def setUp(self):
"""
objects to be tested
"""
self.model1_test = BaseModel()
self.model2_test = BaseModel()
import diagnostics
from models import BaseModel
mod = BaseModel()
print('instantiation successful')
a = 10**4
Ns = [5 * (10**5), a, 5 * a, 10 * a, 50 * a]
diagnostics.runtime_lineplot_N(mod, N_space=Ns, P=4)
print('ahuevooooooooo')
diagnostics.mse_lineplot_N(mod, N_space=Ns, P=4)
def train(args, data):
adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, degreeTasks, neighbor_list = data
features = features.todense()
num_nodes, fea_size = features.shape
num_classes = y_train.shape[1]
placeholders = construct_placeholder(num_nodes, fea_size, num_classes)
model = BaseModel(placeholders,
degreeTasks,
neighbor_list,
num_classes,
fea_size,
hash_dim=args.hash_dim,
hidden_dim=args.hidden_dim,
num_hash=args.n_hash_kernel,
num_layers=args.n_layers)
logits = model.inference()
log_resh = tf.reshape(logits, [-1, num_classes])
lab_resh = tf.reshape(placeholders['labels'], [-1, num_classes])
msk_resh = tf.reshape(placeholders['masks'], [-1])
loss = model.masked_softmax_cross_entropy(log_resh, lab_resh, msk_resh)
accuracy = model.masked_accuracy(log_resh, lab_resh, msk_resh)
train_op = model.training(loss, lr=args.lr, l2_coef=args.weight_decay)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer(),
tf.tables_initializer())
vloss_min = np.inf
vacc_max = 0.0
curr_step = 0
with tf.Session() as sess:
sess.run(init_op)
vacc_early_model = 0.0
vlss_early_model = 0.0
for epoch in range(args.epochs):
train_feed_dict = {}
train_feed_dict.update({placeholders['labels']: y_train})
train_feed_dict.update({placeholders['features']: features})
train_feed_dict.update({placeholders['dropout']: args.dropout})
train_feed_dict.update({placeholders['masks']: train_mask})
_, loss_value_tr, acc_tr = sess.run([train_op, loss, accuracy],
feed_dict=train_feed_dict)
val_feed_dict = {}
val_feed_dict.update({placeholders['labels']: y_val})
val_feed_dict.update({placeholders['features']: features})
val_feed_dict.update({placeholders['dropout']: 0.0})
val_feed_dict.update({placeholders['masks']: val_mask})
loss_value_val, acc_val = sess.run([loss, accuracy],
feed_dict=val_feed_dict)
print(
'Training epoch %d-th: loss = %.5f, acc = %.5f | Val: loss = %.5f, acc = %.5f'
% (epoch + 1, loss_value_tr, acc_tr, loss_value_val, acc_val))
if acc_val >= vacc_max or loss_value_val <= vloss_min:
if acc_val >= vacc_max and loss_value_val <= vloss_min:
vacc_early_model = acc_val
vlss_early_model = loss_value_val
vacc_max = np.max((acc_val, vacc_max))
vloss_min = np.min((loss_value_val, vloss_min))
curr_step = 0
else:
curr_step += 1
if curr_step == args.patience:
print('Early stop! Min loss: ', vloss_min,
', Max accuracy: ', vacc_max)
print('Early stop model validation loss: ',
vlss_early_model, ', accuracy: ', vacc_early_model)
break
test_feed_dict = {}
test_feed_dict.update({placeholders['labels']: y_test})
test_feed_dict.update({placeholders['features']: features})
test_feed_dict.update({placeholders['dropout']: 0.0})
test_feed_dict.update({placeholders['masks']: test_mask})
loss_value_test, acc_test = sess.run([loss, accuracy],
feed_dict=test_feed_dict)
print('Test loss:', loss_value_test, '; Test accuracy:', acc_test)
sess.close()
sys.exit(1)
# get full keras response space on data
refs = []
flattened = []
for point in all_data:
conv_point = np.expand_dims(np.expand_dims(point, axis=2), axis=0)
prob = model.predict_proba(conv_point)[0][0]
refs.append(prob)
flattened.append(point)
refs = np.asarray(refs)
labels_ref = np.concatenate((np.ones(len(sig_data)), np.zeros(len(bkg_data))))
flattened = np.asarray(flattened)
# create drone
drone = BaseModel(len(sig_data[0]), 1)
drone.add_layer(5)
drone.add_layer(1)
conv = BasicConverter(num_epochs=epochNum, threshold=threshold)
drone = conv.convert_model(drone, model, all_data, keras_conv=True)
conv.save_history('./converted_hist.pkl')
drone.save_model('./converted_drone.pkl')
joblib.dump(scaler, open('./scaler_drone.pkl', 'wb'))
refs_drone = []
flattened_drone = []
for point in all_data:
prob = drone.evaluate_total(point)
if args.b:
Settings.encode_bindings = False
if args.t:
Settings.randomize_tokens = False
if args.ro:
Settings.fixed_role_order = False
Settings.training_progress_report = args.v
Settings.combinatorial_generalization_role_fillers = {}
Settings.combinatorial_generalization_role_fillers[args.r] = [0]
comb_test = sets.CombinatorialGeneralization()
if args.m == 0:
model = BaseModel()
train_set = sets.BaseModelTrainSet()
test_set = sets.BaseModelTestSet(test=comb_test)
if args.m == 1:
model = VARSModel()
train_set = sets.VARSTrainSet()
test_set = sets.VARSTestSet(test=comb_test)
for rep in range(1):
if Settings.training_progress_report:
print("Training {} on test role {}".format(model.__class__.__name__,
args.r))
n_epochs, task_acc, vars_acc = model.train(train_set,
from bottle import default_app, route, view, static_file, TEMPLATE_PATH, request, BaseTemplate, debug, hook
from peewee import IntegrityError, DoesNotExist, fn
from bottle_utils.i18n import I18NPlugin
from bottle_utils.i18n import lazy_gettext as _
from input_number import is_valid_number, parse_numbers, get_fingerprint
from models import BaseModel, Number, Place
from configuration import LANGS, MIN_COUNT, MAX_DAYS, DEFAULT_LOCALE, DISPLAY_SIZE
debug(True)
# store database outside of repository so it is not overwritten by git pull
MOD_PATH = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.abspath(os.path.join(MOD_PATH, '../', '../', "lagesonr.db"))
model = BaseModel(database=DB_PATH)
def get_valid_locale(l):
try:
Locale.parse(l)
return l
except UnknownLocaleError:
return DEFAULT_LOCALE
# set as global variable available in all templates (to be able to call e.g. request.locale)
BaseTemplate.defaults['request'] = request
BaseTemplate.defaults['locale_datetime'] = lambda d: format_datetime(
d, format="short", locale=get_valid_locale(request.locale))
# save model
model.save('./keras_jet_conv2d_for_drone.h5')
if not model:
# check if model does exist
print('ERROR: Could not load or create Keras model. Exiting...')
sys.exit(1)
# get full keras response space on data
refs = []
for point in all_data:
prob = model.predict_proba(point)[0][0]
refs.append(prob)
refs = np.asarray(refs)
labels_ref = np.concatenate((np.ones(len(sig_img)), np.zeros(len(bkg_img))))
# create drone
drone = BaseModel(len(sig_img[0].flatten()), 1)
drone.add_layer(675)
drone.add_layer(1)
conv = BasicConverter(num_epochs=epochNum, threshold=threshold)
drone = conv.convert_model(drone, model, all_data, conv_2d=True)
conv.save_history('./converted_hist.pkl')
drone.save_model('./converted_drone.pkl')
refs_drone = []
for point in all_data:
prob = drone.evaluate_total(point)
refs_drone.append(prob)
refs_drone = np.asarray(refs_drone)
labels_drone = np.concatenate((np.ones(len(sig_img)), np.zeros(len(bkg_img))))
def __init__(self):
# Initialize variables
self.support_size = config_settings['support_size'] # support size
self.query_size = config_settings['query_size'] # query size
self.n_epoch = config_settings['n_epoch'] # number of epochs
self.n_inner_loop = config_settings['n_inner_loop'] # number of inner loop
self.batch_size = config_settings['batch_size'] # batch size
self.n_layer = config_settings['n_layer'] # number of layers
self.embedding_dim = config_settings['embedding_dim'] # embedding dimension
self.rho = config_settings['rho'] # local learning rate
self.lamda = config_settings['lambda'] # global learning rate
self.tao = config_settings['tao'] # hyper-parameter to control how much bias term is considered
self.device = torch.device(config_settings['cuda_option'])
self.n_k = config_settings['n_k'] # number of latent factors
self.alpha = config_settings['alpha'] # hyper-parameter to control how much new profile info is added
self.beta = config_settings['beta'] # hyper-parameter to control how much new information is kept
self.gamma = config_settings['gamma'] # hyper-parameter to control how much new preference info is added
self.active_func = config_settings['activation_function'] # choice of activation function
self.rand = config_settings['rand'] # Boolean value to turn on randomization
self.random_state = config_settings['random_state'] # Random seed state
self.split_ratio = config_settings['split_ratio'] # train and test split ratio
# Load dataset
self.train_users, self.test_users = train_test_user_list(
rand=self.rand, random_state=self.random_state, train_test_split_ratio=self.split_ratio
)
self.x1_loading, self.x2_loading = UserLoading(embedding_dim=self.embedding_dim).to(self.device), \
ItemLoading(embedding_dim=self.embedding_dim).to(self.device)
self.n_y = default_info['movielens']['n_y']
# Create user and item embedding matrices
self.UEmb = UserEmbedding(self.n_layer, default_info['movielens']['u_in_dim'] * self.embedding_dim,
self.embedding_dim, activation=self.active_func).to(self.device)
self.IEmb = ItemEmbedding(self.n_layer, default_info['movielens']['i_in_dim'] * self.embedding_dim,
self.embedding_dim, activation=self.active_func).to(self.device)
# Create recommendation model
self.rec_model = RecMAM(self.embedding_dim, self.n_y, self.n_layer, activation=self.active_func).to(self.device)
# Create the full model
self.model = BaseModel(self.x1_loading, self.x2_loading, self.UEmb, self.IEmb, self.rec_model).to(self.device)
# Collect task-specific user, model, and rating weights
self.phi_u, self.phi_i, self.phi_r = self.model.get_weights()
# Create the feature-specific memory component
self.FeatureMem = FeatureMem(self.n_k, default_info['movielens']['u_in_dim'] * self.embedding_dim,
self.model, device=self.device)
# Create the task-specific memory component
self.TaskMem = TaskMem(self.n_k, self.embedding_dim, device=self.device)
# Train the model with meta optimization
self.train = self.train_with_meta_optimization
# Test the model with meta optimization
self.test = self.test_with_meta_optimization
# Train the model
self.train()
# Evaluate the model
self.test()
# save model
model.save('./keras_jet_conv1d_for_drone.h5')
if not model:
# check if model does exist
print('ERROR: Could not load or create Keras model. Exiting...')
sys.exit(1)
# get full keras response space on data
refs = []
for point in all_data:
prob = model.predict_proba(np.expand_dims(point, axis = 0))[0][0]
refs.append(prob)
refs = np.asarray(refs)
labels_ref = np.concatenate((np.ones(len(sig_data)), np.zeros(len(bkg_data))))
# create drone
drone = BaseModel(len(sig_data[0].flatten()), 1)
drone.add_layer(5)
drone.add_layer(1)
conv = BasicConverter(num_epochs = epochNum, threshold = threshold, batch_size = batchSize)
dr_data = all_data
if len(all_data.shape) == 3:
if all_data.shape[1] == 1:
dr_data = np.squeeze(np.moveaxis(all_data, 1, 2), axis = 2)
else:
dr_data = np.squeeze(all_data, axis = 2)
drone = conv.convert_model(drone, model, dr_data, conv_1d = True)
conv.save_history('./converted_hist.pkl')
drone.save_model('./converted_drone.pkl')
device = get_device()
torch.manual_seed(42)
args = parse_args()
import torch.optim as optim
from torch.utils.tensorboard import SummaryWriter
from models import BaseModel
from imdb_data import *
word2id, embeddings = get_glove_embeddings(embedding_size=50)
train_loader, val_loader, test_loader = get_dataloaders(args.batch_size, word2id)
print(args.model)
if args.model == 'basemodel':
net = BaseModel(embeddings, num_classes=2)
net = net.to(device)
# In order to exclude losses computed on null entries (zero),
# set ignore_index=0 for the loss criterion
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=args.lr)
hparams = get_hparams(args, ignore_keys={'no_tensorboard'})
expe_name = get_expe_name(prefix='__IMDB__', hparams=hparams)
# path where to save the model
savepath = Path('./checkpoints/checkpt.pt')
# Tensorboard summary writer
if args.no_tensorboard:
def main(args):
os.makedirs(os.path.join(os.getcwd(), 'logs'), exist_ok=True)
os.makedirs(os.path.join(os.getcwd(), 'models'), exist_ok=True)
os.makedirs(os.path.join(os.getcwd(), 'png'), exist_ok=True)
data_loaders, data_sizes = get_dataloader(
input_dir=args.input_dir,
phases=['train', 'valid'],
batch_size=args.batch_size,
num_workers=args.num_workers)
model = BaseModel(args.bn_mom, args.embed_size, args.num_classes)
model = model.to(device)
criterion = DenseCrossEntropy()
criterion_arc = ArcFaceLoss()
#criterion = nn.CrossEntropyLoss(reduction='sum')
if args.load_model:
checkpoint = torch.load(os.path.join(os.getcwd(), 'models/model.ckpt'))
model.load_state_dict(checkpoint['state_dict'])
params = list(model.parameters())
optimizer = optim.Adam(params, lr=args.learning_rate)
scheduler = lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma)
for epoch in range(args.num_epochs):
for phase in ['train', 'valid']:
since = time.time()
running_loss = 0.0
running_loss_arc = 0.0
running_corrects = 0.0
running_size = 0.0
if phase == 'train':
model.train()
else:
model.eval()
for batch_idx, batch_sample in enumerate(data_loaders[phase]):
optimizer.zero_grad()
image = batch_sample['image']
label = batch_sample['label']
image = image.to(device)
label = label.to(device)
sample_size = image.size(0)
with torch.set_grad_enabled(phase == 'train'):
total_loss = 0.0
output_arc, output = model(image) # [batch_size, num_classes]
_, pred = torch.max(output, 1)
loss = criterion(output, label)
loss_arc = criterion(output_arc, label)
coeff = args.metric_loss_coeff
total_loss = (1 - coeff) * loss + coeff * loss_arc
if phase == 'train':
total_loss.backward()
_ = nn.utils.clip_grad_norm_(model.parameters(), args.clip)
optimizer.step()
running_loss += loss.item()
running_loss_arc += loss_arc.item()
running_corrects += torch.sum(pred == label)
running_size += sample_size
epoch_loss = running_loss / running_size
epoch_loss_arc = running_loss_arc / running_size
epoch_accuracy = float(running_corrects) / running_size
print('| {} SET | Epoch [{:02d}/{:02d}]'.format(phase.upper(), epoch+1, args.num_epochs))
print('\t*- Loss : {:.4f}'.format(epoch_loss))
print('\t*- Loss_Arc : {:.4f}'.format(epoch_loss_arc))
print('\t*- Accuracy : {:.4f}'.format(epoch_accuracy))
# Log the loss in an epoch.
with open(os.path.join(os.getcwd(), 'logs/{}-log-epoch-{:02}.txt').format(phase, epoch+1), 'w') as f:
f.write(str(epoch+1) + '\t' +
str(epoch_loss) + '\t' +
str(epoch_accuracy))
# Save the model check points.
if phase == 'train' and (epoch+1) % args.save_step == 0:
torch.save({'epoch': epoch+1,
'state_dict': model.state_dict()},
os.path.join(os.getcwd(), 'models/model-epoch-{:02d}.ckpt'.format(epoch+1)))
time_elapsed = time.time() - since
print('=> Running time in a epoch: {:.0f}h {:.0f}m {:.0f}s'
.format(time_elapsed // 3600, (time_elapsed % 3600) // 60, time_elapsed % 60))
scheduler.step()
print()
def main(args):
os.makedirs(os.path.join(os.getcwd(), 'logs'), exist_ok=True)
os.makedirs(os.path.join(os.getcwd(), 'models'), exist_ok=True)
# the maximum length of video label in 2nd challenge: 18.
# the maximum length of video label in 3rd challenge: 4.
args.max_vid_label_length = 18 if args.which_challenge == '2nd_challenge' else 4
data_loaders = get_dataloader(
input_dir=args.input_dir,
which_challenge=args.which_challenge,
phases=['train', 'valid'],
max_frame_length=args.max_frame_length,
max_vid_label_length=args.max_vid_label_length,
max_seg_label_length=args.max_seg_label_length,
rgb_feature_size=args.rgb_feature_size,
audio_feature_size=args.audio_feature_size,
batch_size=args.batch_size,
num_workers=args.num_workers)
if args.model_name == 'transformer':
model = GatedTransformerModel(
n_layers=args.n_layers,
n_heads=args.n_heads,
rgb_feature_size=args.rgb_feature_size,
audio_feature_size=args.audio_feature_size,
d_rgb=args.d_rgb,
d_audio=args.d_audio,
d_model=args.d_model,
d_ff=args.d_ff,
d_proj=args.d_proj,
n_attns=args.n_attns,
num_classes=args.num_classes,
dropout=args.dropout)
elif args.model_name == 'gru':
model = GRUModel(n_layers=args.n_layers,
rgb_feature_size=args.rgb_feature_size,
audio_feature_size=args.audio_feature_size,
d_rgb=args.d_rgb,
d_audio=args.d_audio,
d_model=args.d_model,
d_proj=args.d_proj,
n_attns=args.n_attns,
num_classes=args.num_classes,
dropout=args.dropout)
elif args.model_name == 'base':
model = BaseModel(rgb_feature_size=args.rgb_feature_size,
audio_feature_size=args.audio_feature_size,
d_rgb=args.d_rgb,
d_audio=args.d_audio,
d_l=args.d_l,
num_classes=args.num_classes,
dropout=args.dropout)
model = model.to(device)
#center_loss = CenterLoss(num_classes=args.num_classes, feat_dim=args.d_model, device=device)
# This was important from their code.
# Initialize parameters with Glorot / fan_avg.
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
if args.load_model == True:
checkpoint = torch.load(
os.path.join(os.getcwd(),
'models/model-epoch-pretrained-transformer.ckpt'))
model.load_state_dict(checkpoint['state_dict'])
params = list(model.parameters()) #+ list(center_loss.parameters())
optimizer = optim.Adam(params, lr=args.learning_rate)
scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.step_size,
gamma=args.gamma)
for epoch in range(args.num_epochs):
for phase in ['train', 'valid']:
since = time.time()
running_vid_label_loss = 0.0
running_conv_loss = 0.0
running_vid_cent_loss = 0.0
running_seg_time_loss = 0.0
running_vid_label_corrects = 0
running_vid_label_size = 0
running_conv_size = 0
running_seg_time_size = 0
running_num_vid_labels = 0
if phase == 'train':
model.train()
else:
model.eval()
for idx, (_, frame_lengths, frame_rgbs, frame_audios, vid_labels, seg_labels, seg_times) \
in enumerate(data_loaders[phase]):
optimizer.zero_grad()
# frame_lengths: [batch_size]
# frame_rgbs: [batch_size, max_frame_length=300, rgb_feature_size=1024]
# frame_audios: [batch_size, max_frame_length=300, audio_feature_size=128]
# vid_labels: [batch_size, max_vid_label_length]
# seg_labels: [batch_size, max_seg_label_length]
# seg_times: [batch_size, max_seg_label_length]
frame_lengths = frame_lengths.to(device)
frame_rgbs = frame_rgbs.to(device)
frame_audios = frame_audios.to(device)
vid_labels = vid_labels.to(device)
seg_labels = seg_labels.to(device)
seg_times = seg_times.to(device)
batch_size = vid_labels.size(0)
vid_label_size = args.num_classes * batch_size
seg_time_size = batch_size * args.max_seg_label_length * args.max_seg_length
vid_label_lengths = vid_labels.float().ge(0.5).sum(dim=1).view(
-1, 1)
num_vid_labels = vid_labels.float().ge(0.5).sum()
with torch.set_grad_enabled(phase == 'train'):
total_loss = 0.0
vid_label_loss = 0.0
vid_cent_loss = 0.0
seg_time_loss = 0.0
# vid_probs: [batch_size, num_classes]
# attn_idc: [batch_size, num_classes]
# attn_weights: [batch_size, seg_length, n_attns]
# conv_loss: []
if args.model_name == 'transformer':
vid_probs, attn_idc, scores, attn_weights, conv_loss = model(
frame_rgbs, frame_audios, device)
elif args.model_name == 'gru':
vid_probs, attn_idc, scores, attn_weights, conv_loss = model(
frame_rgbs, frame_audios, device)
elif args.model_name == 'base':
vid_probs, attn_weights = model(
frame_rgbs, frame_audios)
vid_label_loss = video_label_loss(vid_probs, vid_labels)
if args.which_challenge == '3rd_challenge':
seg_time_loss = segment_time_loss(
attn_idc, attn_weights, seg_labels, seg_times)
_, vid_preds = torch.topk(vid_probs,
args.max_vid_label_length)
vid_preds = vid_preds + 1
mask = torch.arange(1, args.max_vid_label_length +
1).to(device)
mask = mask.expand(batch_size, args.max_vid_label_length)
zeros = torch.zeros(
batch_size,
args.max_vid_label_length).long().to(device)
vid_preds = torch.where(mask <= vid_label_lengths,
vid_preds, zeros)
vid_preds = torch.zeros(batch_size, args.num_classes +
1).to(device).scatter(
1, vid_preds, 1).long()
vid_preds = vid_preds[:, 1:]
vid_labels = torch.zeros(batch_size, args.num_classes +
1).to(device).scatter(
1, vid_labels, 1).long()
vid_labels = vid_labels[:, 1:]
vid_label_corrects = (vid_labels * vid_preds).sum().float()
total_loss = vid_label_loss / vid_label_size
if args.use_conv_loss == True:
total_loss += conv_loss / batch_size
if args.which_challenge == '3rd_challenge':
total_loss += seg_time_loss / seg_time_size
#total_loss = vid_label_loss / vid_label_size + vid_cent_loss / vid_label_size
if phase == 'train':
total_loss.backward()
_ = nn.utils.clip_grad_norm_(model.parameters(),
args.clip)
optimizer.step()
running_vid_label_loss += vid_label_loss.item()
running_vid_label_corrects += vid_label_corrects.item()
#running_vid_cent_loss += vid_cent_loss.item()
running_vid_label_size += vid_label_size
running_num_vid_labels += num_vid_labels.item()
if args.use_conv_loss == True:
running_conv_loss += conv_loss.item()
running_conv_size += batch_size
if args.which_challenge == '3rd_challenge':
running_seg_time_loss += seg_time_loss.item()
running_seg_time_size += seg_time_size
epoch_vid_label_loss = running_vid_label_loss / running_vid_label_size
epoch_conv_loss = 0.0
epoch_seg_time_loss = 0.0
#epoch_vid_cent_loss = running_vid_cent_loss / running_vid_label_size
epoch_total_loss = epoch_vid_label_loss
if args.use_conv_loss == True:
epoch_conv_loss = running_conv_loss / running_conv_size
epoch_total_loss += epoch_conv_loss
if args.which_challenge == '3rd_challenge':
epoch_seg_time_loss = running_seg_time_loss / running_seg_time_size
epoch_total_loss += epoch_seg_time_loss
#epoch_total_loss = epoch_vid_label_loss + epoch_vid_cent_loss
epoch_vid_label_recall = running_vid_label_corrects / running_num_vid_labels
print('| {} SET | Epoch [{:02d}/{:02d}]'.format(
phase.upper(), epoch + 1, args.num_epochs))
print('\t*- Total Loss : {:.4f}'.format(epoch_total_loss))
print(
'\t*- Video Label Loss : {:.4f}'.format(epoch_vid_label_loss))
if args.use_conv_loss == True:
print(
'\t*- Conv Loss : {:.4f}'.format(epoch_conv_loss))
if args.which_challenge == '3rd_challenge':
print('\t*- Segment Time Loss : {:.4f}'.format(
epoch_seg_time_loss))
print('\t*- Video Label Recall: {:.4f}'.format(
epoch_vid_label_recall))
# Log the loss in an epoch.
with open(
os.path.join(os.getcwd(),
'logs/{}-log-epoch-{:02}.txt').format(
phase, epoch + 1), 'w') as f:
f.write(
str(epoch + 1) + '\t' + str(epoch_total_loss) + '\t' +
str(epoch_vid_label_loss) + '\t' + str(epoch_conv_loss) +
'\t' + str(epoch_seg_time_loss) + '\t' +
str(epoch_vid_label_recall))
# Save the model check points.
if phase == 'train' and (epoch + 1) % args.save_step == 0:
torch.save(
{
'epoch': epoch + 1,
'state_dict': model.state_dict()
},
os.path.join(
os.getcwd(),
'models/model-epoch-{:02d}.ckpt'.format(epoch + 1)))
time_elapsed = time.time() - since
print('=> Running time in a epoch: {:.0f}h {:.0f}m {:.0f}s'.format(
time_elapsed // 3600, (time_elapsed % 3600) // 60,
time_elapsed % 60))
scheduler.step()
print()
def main(args):
since = time.time()
output_dir = os.path.join(os.getcwd(), 'outputs')
os.makedirs(output_dir, exist_ok=True)
data_loaders = get_dataloader(
input_dir=args.input_dir,
which_challenge='3rd_challenge',
phases=['test'],
max_frame_length=args.max_frame_length,
max_vid_label_length=args.max_vid_label_length,
max_seg_label_length=args.max_seg_label_length,
rgb_feature_size=args.rgb_feature_size,
audio_feature_size=args.audio_feature_size,
batch_size=args.batch_size,
num_workers=args.num_workers)
model = BaseModel(rgb_feature_size=args.rgb_feature_size,
audio_feature_size=args.audio_feature_size,
d_rgb=args.d_rgb,
d_audio=args.d_audio,
d_l=args.d_l,
num_classes=args.num_classes,
dropout=args.dropout)
model = model.to(device)
checkpoint = torch.load(
os.path.join(os.getcwd(),
'models/model-epoch-pretrained-base-finetune.ckpt'))
model.load_state_dict(checkpoint['state_dict'])
model.eval()
df_outputs = {i: pd.DataFrame(columns=['vid_id', 'vid_label_pred', 'vid_prob', 'seg_label_pred', 'seg_prob']) \
for i in range(1, args.num_classes+1)}
for idx, (vid_ids, frame_lengths, frame_rgbs, frame_audios, vid_labels, seg_labels, seg_times) \
in enumerate(data_loaders['test']):
if idx % 10 == 0:
print('idx:', idx)
# frame_rgbs: [batch_size, frame_length, rgb_feature_size]
# frame_audios: [batch_size, frame_length, audio_feature_size]
frame_rgbs = frame_rgbs.to(device)
frame_audios = frame_audios.to(device)
batch_size = frame_audios.size(0)
# vid_probs: [batch_size, num_classes]
# attn_weights: [batch_size, max_seg_length]
vid_probs, attn_weights = model(frame_rgbs, frame_audios)
# vid_probs: [batch_size, max_vid_label_length]
# vid_label_preds: [batch_size, max_vid_label_length]
vid_probs, vid_label_preds = torch.topk(vid_probs,
args.max_vid_label_length)
seg_probs, seg_label_preds = torch.topk(attn_weights,
args.seg_pred_length)
vid_probs = vid_probs.cpu().detach().numpy()
vid_label_preds = vid_label_preds.cpu().numpy()
vid_label_preds = vid_label_preds + 1
seg_probs = seg_probs.cpu().detach().numpy()
seg_label_preds = seg_label_preds.cpu().numpy()
seg_label_preds = seg_label_preds + 1
for i in range(batch_size):
for j in range(args.max_vid_label_length):
vid_label_pred = vid_label_preds[i][j]
df_outputs[vid_label_pred] = df_outputs[vid_label_pred].append(
{
'vid_id': vid_ids[i],
'vid_label_pred': vid_label_pred,
'vid_prob': vid_probs[i][j],
'seg_label_pred': list(seg_label_preds[i]),
'seg_prob': list(seg_probs[i])
},
ignore_index=True)
for i in range(1, args.num_classes + 1):
df_outputs[i].to_csv(os.path.join(output_dir, '%04d.csv' % i),
index=False)
time_elapsed = time.time() - since
print('=> Running time in a epoch: {:.0f}h {:.0f}m {:.0f}s'.format(
time_elapsed // 3600, (time_elapsed % 3600) // 60, time_elapsed % 60))