def main():
#prepare images and labels list
images_path = "/home/indranil/action_recognition/test_img/"
label_path = "/home/indranil/action_recognition/labels/"
images_id = []
labels_map = {}
dirs = os.listdir(images_path)
for item in dirs:
image_item = images_path + item
if os.path.isfile(image_item):
label_item = label_path + item.split(".")[0] + ".txt"
if os.path.isfile(label_item):
images_id.append(image_item)
labels_map[image_item] = label_item
#train data loader
train_loader = dataloader.dataLoader(images_id, labels_map, batch_size=16)
#create the refinenet model
net = acnet.acnet().to(device)
# train the network epoch 1-100
optimizer = torch.optim.Adam(net.parameters(),
lr=0.0001,
betas=(0.6, 0.999),
eps=1e-08)
criterion = nn.CrossEntropyLoss().to(device)
train(net=net,
train_loader=train_loader,
criterion=criterion,
optimizer=optimizer,
start_epoch=0,
end_epoch=100,
model_path="/home/indranil/action_recognition/models/",
log_path="/home/indranil/action_recognition/log_dir/")
return
def __init__(self,hyperparameters):
super(Model,self).__init__()
self.device = hyperparameters['device']
self.auxiliary_data_source = hyperparameters['auxiliary_data_source']
self.attr = hyperparameters['attr']
self.all_data_sources = ['resnet_features', 'attributes']
self.DATASET = hyperparameters['dataset']
self.num_shots = hyperparameters['num_shots']
self.latent_size = hyperparameters['latent_size']
self.batch_size = hyperparameters['batch_size']
self.hidden_size_rule = hyperparameters['hidden_size_rule']
self.warmup = hyperparameters['model_specifics']['warmup']
self.generalized = hyperparameters['generalized']
self.classifier_batch_size = 32
#self.img_seen_samples = hyperparameters['samples_per_class'][self.DATASET][0]
#self.att_seen_samples = hyperparameters['samples_per_class'][self.DATASET][1]
#self.att_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][2]
# self.img_unseen_samples = hyperparameters['samples_per_class'][self.DATASET][3]
self.reco_loss_function = hyperparameters['loss']
self.margin = hyperparameters['margin_loss']
self.nepoch = hyperparameters['epochs']
self.lr_cls = hyperparameters['lr_cls']
self.cross_reconstruction = hyperparameters['model_specifics']['cross_reconstruction']
self.cls_train_epochs = hyperparameters['cls_train_steps']
#self.dataset = dataloader(self.DATASET, copy.deepcopy(self.auxiliary_data_source) , device= 'cuda')
self.dataset = dataLoader(copy.deepcopy(self.auxiliary_data_source) , device= 'cuda', attr = self.attr)
if self.DATASET=='CUB':
self.num_classes=200
self.num_novel_classes = 50
elif self.DATASET=='SUN':
self.num_classes=717
self.num_novel_classes = 72
elif self.DATASET=='AWA1' or self.DATASET=='AWA2':
self.num_classes=50
self.num_novel_classes = 10
if self.attr == 'attributes':
feature_dimensions = [2048, self.dataset.K]
elif self.attr == 'bert':
feature_dimensions = [2048, 768] #2048, 768
# Here, the encoders and decoders for all modalities are created and put into dict
self.fc_ft = nn.Linear(2048,2048)
self.fc_ft.to(self.device)
self.ft_bn = nn.BatchNorm1d(2048).to(self.device)
self.fc_at = nn.Linear(self.dataset.K, self.dataset.K)
self.fc_at.to(self.device)
self.at_bn = nn.BatchNorm1d(self.dataset.K).to(self.device)
self.encoder = {}
for datatype, dim in zip(self.all_data_sources,feature_dimensions):
self.encoder[datatype] = models.encoder_template(dim,self.latent_size,self.hidden_size_rule[datatype],self.device)
print(str(datatype) + ' ' + str(dim))
self.decoder = {}
for datatype, dim in zip(self.all_data_sources,feature_dimensions):
self.decoder[datatype] = models.decoder_template(self.latent_size,dim,self.hidden_size_rule[datatype],self.device)
# An optimizer for all encoders and decoders is defined here
parameters_to_optimize = list(self.parameters())
for datatype in self.all_data_sources:
parameters_to_optimize += list(self.encoder[datatype].parameters())
parameters_to_optimize += list(self.decoder[datatype].parameters())
parameters_to_optimize += list(self.fc_ft.parameters())
parameters_to_optimize += list(self.fc_at.parameters())
parameters_to_optimize += list(self.ft_bn.parameters())
parameters_to_optimize += list(self.at_bn.parameters())
self.optimizer = optim.Adam( parameters_to_optimize ,lr=hyperparameters['lr_gen_model'], betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True)
if self.reco_loss_function=='l2':
self.reconstruction_criterion = nn.MSELoss(size_average=False)
elif self.reco_loss_function=='l1':
self.reconstruction_criterion = nn.L1Loss(size_average=False)
self.reparameterize_with_noise = True
def TRAIN_MODEL(modelname):
# Load all the data
image, text, input_len, label_len, original_text, max_len = dataLoader(
).prepareData(iam=True, cvl=False)
if modelname == 'lstm':
model, model_arch = MODEL_ARCHITECTURE().CNN_BiLSTM(max_len)
else:
model, model_arch = MODEL_ARCHITECTURE().CNN_BiGRU(max_len)
# Compile loaded model
try:
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer='adam')
except:
print('ERROR IN MODEL COMPILATION')
# Create model checkpoint
filepath = 'best_CNN_LSTM_model_final.hdf5'
checkpoint = ModelCheckpoint(filepath=filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
callback_list = [checkpoint]
#Create train and validation data
new_image,new_text,new_input_len,new_label_len,new_original_text =[],[],[],[],[]
for index, len_ in enumerate(input_len):
if label_len[index] == len_ and len_ <= 19 and label_len[index] != 0:
new_image.append(image[index])
new_text.append(text[index])
new_input_len.append(31)
new_label_len.append(label_len[index])
new_original_text.append(original_text[index])
print('DATA LOADED SUCESSFULLY')
# Select 10% data for validation and 90% for training
total_train_len = int(abs((len(image) * 90) / 100))
print("TOTAL LENGTH OD TRAINING DATA {}".format(total_train_len))
train_image = np.array(new_image[:total_train_len])
train_text = np.array(new_text[:total_train_len])
train_input_len = np.array(new_input_len[:total_train_len])
train_label_len = np.array(new_label_len[:total_train_len])
#original_trainlabel = np.array(new_original_text[:total_train_len])
valid_image = np.array(new_image[total_train_len + 1:])
valid_text = np.array(new_text[total_train_len + 1:])
valid_input_len = np.array(new_input_len[total_train_len + 1:])
valid_label_len = np.array(new_label_len[total_train_len + 1:])
#original_valid_label = np.array(new_original_text[total_train_len+1:])
char_list = string.ascii_letters + string.digits
train_padded_text = pad_sequences(train_text,
maxlen=max_len,
padding='post',
value=len(char_list))
valid_padded_text = pad_sequences(valid_text,
maxlen=max_len,
padding='post',
value=len(char_list))
# Train the model
model.fit(
x=[train_image, train_padded_text, train_input_len, train_label_len],
y=np.zeros(len(train_image)),
batch_size=config.BATCH_SIZE,
epochs=config.EPOCHS,
validation_data=([
valid_image, valid_padded_text, valid_input_len, valid_label_len
], [np.zeros(len(valid_image))]),
verbose=1,
callbacks=callback_list)
# Save the trained_model in disk
model.save_weights("best_CNN_LSTM_model_final_2.h5")
model_json = model_arch.to_json()
with open("CNN_LSTM_FINAL_2.json", "w") as json_file:
json_file.write(model_json)
print("MODEL SAVED SUCESSFULLY ")
def main():
if not sys.warnoptions:
warnings.simplefilter("ignore")
# --- hyper parameters --- #
BATCH_SIZE = 256
LR = 1e-3
WEIGHT_DECAY = 1e-4
N_layer = 18
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# --- data process --- #
# info
src_path = './data/'
target_path = './saved/ResNet18/'
model_path = target_path + 'pkls/'
pred_path = target_path + 'preds/'
if not os.path.exists(model_path):
os.makedirs(model_path)
if not os.path.exists(pred_path):
os.makedirs(pred_path)
# evaluation: num of classify labels & image size
# output testing id csv
label2num_dict, num2label_dict = data_evaluation(src_path)
# load
train_data = dataLoader(src_path, 'train', label2num_dict)
train_len = len(train_data)
test_data = dataLoader(src_path, 'test')
train_loader = Data.DataLoader(
dataset=train_data,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=12,
)
test_loader = Data.DataLoader(
dataset=test_data,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=12,
)
# --- model training --- #
# fp: for storing data
fp_train_acc = open(target_path + 'train_acc.txt', 'w')
fp_time = open(target_path + 'time.txt', 'w')
# train
highest_acc, train_acc_seq = 0, []
loss_funct = nn.CrossEntropyLoss()
net = ResNet(N_layer).to(device)
optimizer = torch.optim.Adam(net.parameters(),
lr=LR,
weight_decay=WEIGHT_DECAY)
print(net)
for epoch_i in count(1):
right_count = 0
# print('\nTraining epoch {}...'.format(epoch_i))
# for batch_x, batch_y in tqdm(train_loader):
for batch_x, batch_y in train_loader:
batch_x = batch_x.to(device)
batch_y = batch_y.to(device)
# clear gradient
optimizer.zero_grad()
# forward & backward
output = net.forward(batch_x.float())
highest_out = torch.max(output, 1)[1]
right_count += sum(batch_y == highest_out).item()
loss = loss_funct(output, batch_y)
loss.backward()
# update parameters
optimizer.step()
# calculate accuracy
train_acc = right_count / train_len
train_acc_seq.append(train_acc * 100)
if train_acc > highest_acc:
highest_acc = train_acc
# save model
torch.save(
net.state_dict(),
'{}{}_{}_{}.pkl'.format(model_path,
target_path.split('/')[2],
round(train_acc * 1000), epoch_i))
# write data
fp_train_acc.write(str(train_acc * 100) + '\n')
fp_time.write(
str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + '\n')
print('\n{} Epoch {}, Training accuracy: {}'.format(
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), epoch_i,
train_acc))
# test
net.eval()
test_df = pd.read_csv(src_path + 'testing_data/testing_labels.csv')
with torch.no_grad():
for i, (batch_x, _) in enumerate(test_loader):
batch_x = batch_x.to(device)
output = net.forward(batch_x.float())
highest_out = torch.max(output, 1)[1].cpu()
labels = [
num2label_dict[out_j.item()] for out_j in highest_out
]
test_df['label'].iloc[i * BATCH_SIZE:(i + 1) *
BATCH_SIZE] = labels
test_df.to_csv('{}{}_{}_{}.csv'.format(pred_path,
target_path.split('/')[2],
round(train_acc * 1000),
epoch_i),
index=False)
net.train()
lr_decay(optimizer)
fp_train_acc.close()
fp_time.close()
model.cuda()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(0.9, 0.999),
weight_decay=args.reg)
criteration = nn.CrossEntropyLoss()
print('cross-validation: {}'.format(i))
train_loader = data_utils.DataLoader(
dataLoader(
train_name=train_names[i],
test_name=test_names[i],
img_info=img_info,
train=True,
data_transforms=transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
# transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])),
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
**loader_kwargs)
test_loader = data_utils.DataLoader(
dataLoader(
train_name=train_names[i],
test_name=test_names[i],
img_info=img_info,
from numpy import array from preprocess import Preprocess from dataloader import dataLoader from sklearn.feature_extraction.text import TfidfVectorizer from random import random from numpy import cumsum from keras.layers import TimeDistributed from keras.preprocessing.sequence import pad_sequences from sklearn.decomposition import TruncatedSVD import h5py os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' train_filename = "msr_paraphrase_train.txt" test_filename = "msr_paraphrase_test.txt" train_labels, train_pair1, train_pair2 = dataLoader(train_filename) test_labels, test_pair1, test_pair2 = dataLoader(test_filename) assert len(train_pair1) == len(train_pair2) assert len(test_pair1) == len(test_pair2) text = train_pair2 + train_pair1 text = " ".join(text) preprocess = Preprocess(bigrams=False, vocab_size=8000) preprocess.build_vocab(text, stem=True) n = -1 train_pair1 = preprocess.preprocess(train_pair1)[:n] train_pair2 = preprocess.preprocess(train_pair2)[:n] test_pair1 = preprocess.preprocess(test_pair1)[:n] test_pair2 = preprocess.preprocess(test_pair2)[:n]
if len(sys.argv) != 4:
print("Number of augments is wrong! Should get 4 but get %s instead \n" %
(len(sys.argv)))
exit(1)
else:
in_path = sys.argv[1]
out_path = sys.argv[2]
input_len = int(sys.argv[3])
if input_len != 720 and input_len != 1440:
print("Wrong length of input, can only be 720 or 1440")
exit(1)
dataset = dl.dataLoader(in_path)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=200,
shuffle=True,
num_workers=1,
pin_memory=True)
net = md.CNN(input_len).to(device)
criterion = nn.BCEWithLogitsLoss(pos_weight=torch.FloatTensor([9.0]))
learning_rate = 1e-4
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
print_interval = 10
num_epochs = 4
for epoch in range(num_epochs):
for i, (frame) in enumerate(train_loader):
ACC = (TP + TN) / (TP + FN + TN + FP)
TPR = TP / (TP + FN)
TNR = TN / (TN + FP)
print(
'\nTest Set, Loss: {:.4f}, Test error: {:.4f}, ACC: {:.4f}, TPR: {:.4f}, TNR: {:.4f}'
.format(test_loss.cpu().numpy()[0], test_error, ACC, TPR, TNR))
if __name__ == "__main__":
for i in range(5):
print('cross-validation: {}'.format(i))
num_in_train, num_in_test = util_data.generate_train_test_txt(
"./data", train_bag_name[i], test_bag_name[i], img_name, img_path,
img_label)
train_loader = data_utils.DataLoader(dataLoader(
bag_name=train_bag_name[i], ins_num=num_in_train, train=True),
batch_size=1,
shuffle=True,
**loader_kwargs)
test_loader = data_utils.DataLoader(dataLoader(
bag_name=test_bag_name[i], ins_num=num_in_test, train=False),
batch_size=1,
shuffle=False,
**loader_kwargs)
print('Start Training')
for epoch in range(1, args.epochs + 1):
train(epoch)
print('Start Testing')
test()
