def resnet50(load):
"""Constructs a ResNet-50 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(Bottleneck, [3, 4, 6, 3])
if load == True:
utils.load_net(model, "checkpoint.tar")
return model
def run(self):
if self.train_reward_also:
self.reward_trainer.start()
reward_net_path = self.reward_trainer.get_reward_net_folder()
if self.policy_net_key is None:
self.policy_net_key = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
train_policy(self.environment, reward_net_arg=reward_net_path, policy_net_key=self.policy_net_key, callbacks=self.policy_callbacks)
else:
episodes = self.policy.max_episodes - self.policy.episode
self.policy.fit(episodes, reward_loader=lambda: load_net(os.path.join(rewards_dir(), self.environment, self.policy.reward_net_key), True), callbacks=self.policy_callbacks)
def MCTS_self_play(iter, num_games, workers=1):
if not os.path.exists('datasets/iter{}'.format(iter)):
os.makedirs('datasets/iter{}'.format(iter))
net = load_net(iter)
if workers > 1:
net.share_memory()
mp.set_start_method("spawn", force=True)
net.eval()
if workers > 1:
process = []
for i in range(workers):
w = mp.Process(target=worker, args=(iter, num_games, net, i))
w.start()
process.append(w)
for w in process:
w.join()
else:
worker(iter, num_games, net)
def sliderChanged(self):
episodes_for_checkpoint = get_episodes_for_checkpoint()
s_val = self.ui.sliderTraining.value()
# self.setCursor(QtGui.QCursor(QtCore.Qt.PointingHandCursor))
policy_name = policies_dir()
policies = [
int(el.split('-')[1].split('.')[0]) for el in os.listdir(
os.path.join(policy_name, self.environment, self.agent_key))
if el.endswith('.pth')
]
# new value is bigger than max value
if s_val > int(max(policies) / episodes_for_checkpoint):
s_val = int(max(policies) / episodes_for_checkpoint)
self.ui.sliderTraining.setValue(s_val)
policy_net = os.path.join(
policy_name, self.environment, self.agent_key,
'policy_net-' + str(s_val * episodes_for_checkpoint) + '.pth')
new_agent = load_net(policy_net)
if new_agent.episode != self.game_thread.agent:
self.game_thread.set_agent(new_agent)
from run_SiamRPN import SiamRPN_init, SiamRPN_track
from utils import get_axis_aligned_bbox, cxy_wh_2_rect, load_net
parser = argparse.ArgumentParser(description='PyTorch SiameseX demo')
parser.add_argument('--model',
metavar='model',
default='SiamRPNPPRes50',
type=str,
help='which model to use.')
args = parser.parse_args()
# load net
net = eval(args.model)()
load_net('./{}.pth'.format(args.model), net)
net.eval().cuda()
# image and init box
image_files = sorted(glob.glob('./testData/*.jpg'))
init_rbox = [3641, 1778, 3810, 1778, 3810, 2313, 3641, 2313]
[cx, cy, w, h] = get_axis_aligned_bbox(init_rbox)
# tracker init
target_pos, target_sz = np.array([cx, cy]), np.array([w, h])
im = cv2.imread(image_files[0]) # HxWxC
state = SiamRPN_init(im, target_pos, target_sz, net, args.model)
# tracking and visualization
toc = 0
for f, image_file in enumerate(image_files):
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
dataset_folder = 'imSitu'
imgset_folder = 'resized_256'
print(
'model spec :, mac net v pred for training and loss calc normalizing from only matching role count '
)
train_set = json.load(open(dataset_folder + "/train.json"))
encoder = imsitu_encoder(train_set)
model = mac_model_with_verb.E2ENetwork(encoder, args.gpuid)
# To group up the features
cnn_features, verb_features, role_features = utils.group_features_single(
model)
train_set = imsitu_loader(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader(imgset_folder, dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader(imgset_folder, traindev_set, encoder,
model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer_single(lr, weight_decay, optimizer_select,
cnn_features, verb_features,
role_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
'''optimizer = torch.optim.Adam([{'params': model.conv.parameters(), 'lr': 5e-5},
{'params': model.verb.parameters(), 'lr': 5e-5},
{'params': model.role_labeller.parameters()}],
lr=1e-3)'''
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=lr_step, gamma=lr_gamma)
#gradient clipping, grad check
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, optimizer,
scheduler, n_epoch, args.output_dir, encoder, args.gpuid, clip_norm,
lr_max, model_name, args)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.benchmark = True
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
train_dset = VQAFeatureDataset('train', dictionary)
eval_dset = VQAFeatureDataset('val', dictionary)
batch_size = args.batch_size
constructor = 'build_%s' % args.model
model = getattr(base_model, constructor)(train_dset, args.num_hid).cuda()
model = nn.DataParallel(model).cuda()
train_loader = DataLoader(train_dset, batch_size, shuffle=True, num_workers=1)
eval_loader = DataLoader(eval_dset, batch_size, shuffle=True, num_workers=1)
utils.load_net(args.pretrained_model, [model])
eval_score, bound, pred_all, qIds = evaluate(model, eval_loader)
print('eval score: %.2f (%.2f)' % (100 * eval_score, 100 * bound))
results = make_json(pred_all, qIds, eval_loader)
with open(args.output+'/%s_%s.json' \
% ('eval', args.model), 'w') as f:
json.dump(results, f)
def main(argv):
# folder for saving
subfold = argv[1]
if not os.path.exists(subfold):
os.mkdir(subfold)
os.mkdir(os.path.join(subfold, "features"))
# load data
X_cnn_raw = []
labels_cnn_raw = []
num_grp = 3
for i in range(num_grp):
fname = '../LRG-fits/data/lrg_171020/sample-lrg-train-120-120-c3-gr{0}.pkl'.format(
i)
with open(fname, 'rb') as fp:
datadict = pickle.load(fp)
X_cnn_raw.append(datadict['data'])
labels_cnn_raw.append(datadict['label'])
time.sleep(3)
X_test = []
labels_test = []
fname = '../LRG-fits/data/lrg_171020/sample-lrg-test-120-120-c3.pkl'
with open(fname, 'rb') as fp:
datadict = pickle.load(fp)
X_test.append(datadict['data'])
labels_test.append(datadict['label'])
# Combine and normalization
sample_mat = np.vstack(X_cnn_raw)
del (X_cnn_raw)
labels_cnn = np.hstack(labels_cnn_raw)
del (labels_cnn_raw)
# sample_mat = np.nan_to_num(sample_mat)
rs = 120
'''
with open("../nets/norm_params.pkl", 'rb') as fp:
normparam = pickle.load(fp)
X_max = normparam["X_max"]
X_min = normparam["X_min"]
X_mean = normparam["X_mean"]
X_train_cnn = (sample_mat - X_min) / (X_max - X_min)
# X_norm = sample_mat
X_w_cnn = X_train_cnn - X_mean
X_tr_cnn = X_w_cnn.reshape(-1, rs, rs, 1).astype('float32')
'''
X_tr_cnn = sample_mat.reshape(-1, rs, rs, 1).astype('float32')
idx = np.random.permutation(len(labels_cnn))
numsamples = 100000
X_in = X_tr_cnn[idx[0:numsamples], :, :, :]
# get labels
X_out = labels_cnn[idx[0:numsamples]].astype('int32')
mask_layer1 = [100, 100, 100, 100, 0, 1, 100, 100]
data_layer1, label_layer1 = sub2triple(data=X_in,
label=X_out,
mask=mask_layer1)
label_layer1_hotpot = vec2onehot(label=label_layer1, numclass=2)
numclass = 2
encode_nodes = 64
cnn = ConvNet.ConvNet(input_shape=data_layer1.shape,
kernel_size=[3, 3, 3, 3, 3],
kernel_num=[8, 8, 16, 32, 32],
fc_nodes=[],
encode_nodes=encode_nodes,
padding=('SAME', 'SAME'),
stride=(2, 2),
numclass=numclass,
sess=None,
name=None)
cnn.cae_build()
cnn.cnn_build(learning_rate=0.001) # In order to init the weights
foldname = "./nets/pretrain-171020-2cls/"
name = "pretrain-120-171020-2cls.pkl"
cnn.sess, cnn.name = utils.load_net(os.path.join(foldname, name))
# train
num_epochs = 100
learning_rate = 0.001
batch_size = 100
droprate = 0.5
cnn.cnn_train(data=data_layer1,
label=label_layer1_hotpot,
num_epochs=num_epochs,
learning_rate=learning_rate,
batch_size=batch_size,
droprate=droprate)
# save features
fname = "code_l5.pkl"
folder = "{0}/features/".format(subfold)
if not os.path.exists(folder):
os.mkdir(folder)
numsample = data_layer1.shape[0]
numone = numsample // 10
code = np.zeros((numsample, encode_nodes))
for i in range(10):
code[i * numone:(i + 1) * numone] = cnn.cae_encode(
data_layer1[i * numone:(i + 1) * numone, :, :, :])
# code = cnn.cae_encode(data_layer1)
label = label_layer1
with open(os.path.join(folder, fname), 'wb') as fp:
code_dict = {"code": code, "label": label}
pickle.dump(code_dict, fp)
# save net
foldname = "{0}/net_l5_140".format(subfold)
name = "net_l5.pkl"
netname = "model-l5.ckpt"
if os.path.exists(foldname):
os.system("rm -r %s" % (foldname))
os.mkdir(foldname)
cnn.cnn_save(namepath=os.path.join(foldname, name),
netpath=os.path.join(foldname, netname))
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--role_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--dataset_folder',
type=str,
default='./imSitu',
help='Location of annotations')
parser.add_argument('--imgset_dir',
type=str,
default='./resized_256',
help='Location of original images')
parser.add_argument('--frcnn_feat_dir',
type=str,
help='Location of output from detectron')
parser.add_argument('--batch_size', type=int, default=64)
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = args.batch_size
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
#dataset_folder = 'imSitu'
#imgset_folder = 'resized_256'
dataset_folder = args.dataset_folder
imgset_folder = args.imgset_dir
train_set = json.load(open(dataset_folder + "/train_new_2000_all.json"))
imsitu_roleq = json.load(open("imsitu_data/imsitu_questions_prev.json"))
encoder = imsitu_encoder(train_set, imsitu_roleq)
model = model_vgg_featextractor4ggnn.BaseModel(encoder, args.gpuid)
# To group up the features
train_set = imsitu_loader_resnet_featextract(imgset_folder, train_set,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size,
shuffle=False,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev_new_2000_all.json"))
dev_set = imsitu_loader_resnet_featextract(imgset_folder, dev_set,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=batch_size,
shuffle=False,
num_workers=n_worker)
test_set = json.load(open(dataset_folder + "/test_new_2000_all.json"))
test_set = imsitu_loader_resnet_featextract(imgset_folder, test_set,
model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=batch_size,
shuffle=False,
num_workers=n_worker)
utils.set_trainable(model, False)
'''if args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])'''
#load verb and role modules
utils.load_net(args.verb_module, [model.conv_verbs], ['conv'])
utils.load_net(args.role_module, [model.conv_nouns], ['conv'])
if args.gpuid >= 0:
model.cuda()
extract_features(model, [train_loader, dev_loader, test_loader],
args.gpuid)
'''print('rechecking')
{
'params': model.sharedNet.parameters()
},
{
'params': model.fc.parameters(),
'lr': 10 * LEARNING_RATE
},
],
lr=LEARNING_RATE,
momentum=MOMENTUM)
if CUDA:
model = model.cuda()
if args.load is not None:
utils.load_net(model, args.load)
else:
load_pretrained(model.sharedNet)
training_statistic = []
testing_s_statistic = []
testing_t_statistic = []
for e in range(0, EPOCHS):
_lambda = (e + 1) / EPOCHS
# _lambda = 0.0
res = train(model, optimizer, e + 1, _lambda)
print('###EPOCH {}: Class: {:.6f}, CORAL: {:.6f}, Total_Loss: {:.6f}'.
format(
e + 1,
sum(row['classification_loss'] / row['total_steps']
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Eval network')
parser.add_argument('--net',
help='name of the network file',
required=True)
parser.add_argument('--anchors', help='network anchor file', required=True)
parser.add_argument('--test-file',
help='name of the test file',
required=True)
parser.add_argument('--output',
help='name of the output',
default='res/out.pickle')
args = parser.parse_args()
path = setup_path(args)
setup_logging(path=path, parser=parser)
gamma, net = load_net(args.net)
an = tc.load_sframe(args.anchors)
logging.info(f"Reading test file: {args.test_file}")
test = tc.load_sframe(args.test_file)
logging.info('Starting evaluation')
res = eval_net(net=net, anchors=an, data=test)
logging.info('Storing results')
with open(f"{path}/{gamma}-evalresults.pickle", 'wb+') as f:
pickle.dump(res, f)
help='name of the function file',
required=True,
default='/storage/users/cnalab/apkdata-tanya/binary/new.large.sframe')
parser.add_argument(
'--net', help='name of a network file (extraction only for anchors)')
parser.add_argument('--output', help='output path', required=True)
args = parser.parse_args()
#test_file = '/storage/users/cnalab/apkdata-tanya/binary/test-tc-1000.npy'
#if test_file:
# print(f"Reading test file: {test_file}")
# test_apns = np.load(test_file)
path = setup_path(args=args)
setup_logging(path=path, parser=parser)
net_file = args.net
logging.info(f"Reading reading net file {net_file}")
gamma, net = load_net(net_file)
test_apns = list(net.keys())
logging.info(f"Extracted apn: {len(test_apns)}")
setup_turi()
tc.config.set_runtime_config('TURI_DEFAULT_NUM_PYLAMBDA_WORKERS', 16)
logging.info('Loading functions')
mw = load_functions_partition(directory='', name=args.functions)
logging.info('Filter started')
test_f = mw.filter_by(values=test_apns, column_name='apk')
test_f.save(f"{path}/convert-out-{len(test_apns)}", format='binary')
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
dataset_folder = 'imSitu'
imgset_folder = 'resized_256'
print(
'model spec :, 256 hidden, 1e-4 init lr, 25 epoch decay, 4 layer mlp for g,2mlp f1, 3 att layers with res connections param init xavier uni 2 heads dropout 0.5 mask 6loss maskb4g transformopt'
)
train_set = json.load(open(dataset_folder + "/train.json"))
encoder = imsitu_encoder(train_set)
model = model_vsrl_finetune_selfatt_ff.RelationNetworks(
encoder, args.gpuid)
# To group up the features
cnn_features, verb_features, role_features = utils.group_features(model)
train_set = imsitu_loader(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=24,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader(imgset_folder, dev_set, encoder,
model.train_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=24,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader(imgset_folder, traindev_set, encoder,
model.train_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer(lr, weight_decay, optimizer_select,
cnn_features, verb_features, role_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
opt = utils.NoamOpt(256, 1, 4000, optimizer)
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=lr_step,
gamma=lr_gamma)
#gradient clipping, grad check
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, opt, scheduler,
n_epoch, args.output_dir, encoder, args.gpuid, clip_norm, lr_max,
model_name, args)
torch.backends.cudnn.benchmark = True
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
train_dset = VQAFeatureDataset_withmask('train', dictionary)
eval_dset = VQAFeatureDataset_withmask('val', dictionary)
batch_size = args.batch_size
constructor = 'build_baseline0_newatt'
baseline = getattr(base_model, constructor)(train_dset, args.num_hid)
constructor = 'build_%s' % args.model
model = getattr(pretrained_tda_caq_model,
constructor)(train_dset, args.num_hid, baseline).cuda()
model.w_emb.init_embedding('data/glove6b_init_300d.npy')
utils.load_net(args.pretrained_tda_model, [model.tda_model], ['module'])
utils.set_trainable(model.tda_model, False)
model = nn.DataParallel(model).cuda()
#seventyfive = list(range(0, int(math.ceil(len(train_dset) * 0.75))))
#trainset_1 = torch.utils.data.Subset(train_dset, seventyfive)
train_loader = DataLoader(train_dset,
batch_size,
shuffle=True,
num_workers=1)
eval_loader = DataLoader(eval_dset,
batch_size,
shuffle=True,
def main():
import argparse
parser = argparse.ArgumentParser(
description="VQA using MAC. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--imgset_folder',
type=str,
default='',
help='path to image set')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
dataset_folder = 'vqa_data'
imgset_folder = args.imgset_folder
print('model spec :, mac net for vqa 2.0')
train_set = json.load(open(dataset_folder + "/vqa_openended_train.json"))
encoder = vqa_encoder(train_set)
model = model_mac_vqa.E2ENetwork(encoder, args.gpuid)
# To group up the features
cnn_features, mac_features = utils.group_features_vqa(model)
train_set = vqa_loader(imgset_folder + '/train2014', train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker,
collate_fn=collate_data)
dev_set = json.load(open(dataset_folder + "/vqa_openended_val.json"))
dev_set = vqa_loader(imgset_folder + '/val2014', dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker,
collate_fn=collate_data)
'''test_set = json.load(open(dataset_folder +"/test.json"))
test_set = imsitu_loader_roleq(imgset_folder, test_set, encoder, model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set, batch_size=64, shuffle=True, num_workers=n_worker)
traindev_set = json.load(open(dataset_folder +"/dev.json"))
traindev_set = imsitu_loader_roleq(imgset_folder, traindev_set, encoder, model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set, batch_size=8, shuffle=True, num_workers=n_worker)'''
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer_vqa(optimizer_select, cnn_features,
mac_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=lr_step, gamma=lr_gamma)
#gradient clipping, grad check
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
if args.evaluate:
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"]
avg_score /= 8
print('Dev average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#write results to csv file
role_dict = top1.role_dict
fail_val_all = top1.value_all_dict
with open('role_pred_data.json', 'w') as fp:
json.dump(role_dict, fp, indent=4)
with open('fail_val_all.json', 'w') as fp:
json.dump(fail_val_all, fp, indent=4)
print('Writing predictions to file completed !')
elif args.test:
top1, top5, val_loss = eval(model,
test_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"]
avg_score /= 8
print('Test average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
else:
print('Model training started!')
train(model, train_loader, dev_loader, None, optimizer, scheduler,
n_epoch, args.output_dir, encoder, args.gpuid, clip_norm, lr_max,
model_name, args)
def main(args=None):
parser = argparse.ArgumentParser(
description='Simple training script for training a RetinaNet network.')
parser.add_argument(
'--train-file',
help='Path to file containing training annotations (see readme)')
parser.add_argument(
'--pretrained_model',
help='Path to file containing training annotations (see readme)')
parser.add_argument('--classes-file',
help='Path to file containing class list (see readme)')
parser.add_argument(
'--val-file',
help=
'Path to file containing validation annotations (optional, see readme)'
)
parser.add_argument('--role-file', help='Path to role file')
parser.add_argument('--epochs',
help='Number of epochs',
type=int,
default=50)
parser.add_argument('--title', type=str, default='')
parser.add_argument("--resume-epoch", type=int, default=0)
parser.add_argument("--detach-epoch", type=int, default=12)
parser.add_argument("--gt-noun-epoch", type=int, default=5)
parser.add_argument("--hidden-size", type=int, default=1024)
parser.add_argument("--lr-decrease", type=int, default=10)
parser.add_argument("--second-lr-decrease", type=int, default=20)
parser.add_argument("--iteration", type=float, default=100.0)
parser.add_argument("--lr", type=float, default=.0006)
parser.add_argument("--batch-size", type=int, default=16)
parser = parser.parse_args(args)
writer, log_dir = init_log_dir(parser)
print('correct version')
print("loading dev")
with open('./SWiG_jsons/dev.json') as f:
dev_gt = json.load(f)
print("loading imsitu_dpace")
with open('./SWiG_jsons/imsitu_space.json') as f:
all = json.load(f)
verb_orders = all['verbs']
noun_dict = all['nouns']
dataloader_train, dataset_train, dataloader_val, dataset_val = init_data(
parser, verb_orders)
print("loading model")
retinanet = model_new.resnet50(num_classes=dataset_train.num_classes(),
num_nouns=dataset_train.num_nouns(),
parser=parser,
pretrained=True)
utils.load_net(parser.pretrained_model, [retinanet])
print('Loading pretrained RetinaNet finished!')
utils.set_trainable(retinanet, False)
utils.set_trainable(retinanet.vocab_linear, True)
utils.set_trainable(retinanet.vocab_linear_2, True)
utils.set_trainable(retinanet.verb_embeding, True)
utils.set_trainable(retinanet.noun_embedding, True)
utils.set_trainable(retinanet.regressionModel, True)
utils.set_trainable(retinanet.classificationModel, True)
utils.set_trainable(retinanet.rnn, True)
utils.set_trainable(retinanet.rnn_linear, True)
optimizer = torch.optim.Adamax([
{
'params': retinanet.vocab_linear.parameters()
},
{
'params': retinanet.vocab_linear_2.parameters()
},
{
'params': retinanet.verb_embeding.parameters()
},
{
'params': retinanet.noun_embedding.parameters()
},
{
'params': retinanet.regressionModel.parameters()
},
{
'params': retinanet.classificationModel.parameters()
},
{
'params': retinanet.rnn.parameters()
},
{
'params': retinanet.rnn_linear.parameters()
},
],
lr=1e-3)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
#retinanet = torch.nn.DataParallel(retinanet).cuda()
retinanet = retinanet.cuda()
#optimizer = optim.Adam(retinanet.parameters(), lr=parser.lr)
print('weights loaded')
best_eval = 0
for epoch_num in range(parser.resume_epoch, parser.epochs):
train(retinanet, optimizer, dataloader_train, parser, epoch_num,
writer)
#torch.save({'state_dict': retinanet.module.state_dict(), 'optimizer': optimizer.state_dict()}, log_dir + '/checkpoints/retinanet_{}.pth'.format(epoch_num))
print('Evaluating dataset')
eval_avg = evaluate(retinanet, dataloader_val, parser, dataset_val,
dataset_train, verb_orders, dev_gt, epoch_num,
writer, noun_dict)
if eval_avg > best_eval:
print('New best model at epoch ', epoch_num)
scheduler.step()
from net import SiamRPNPPRes50, SiamRPNResNeXt22
from run_SiamRPN import SiamRPN_init, SiamRPN_track
from utils import get_axis_aligned_bbox, cxy_wh_2_rect, load_net
parser = argparse.ArgumentParser(description='PyTorch SiameseX demo')
parser.add_argument('--model',
metavar='model',
default='SiamRPNResNeXt22',
type=str,
help='which model to use.')
args = parser.parse_args()
# load net
net = eval(args.model)()
load_net('../cp/temp/{}.pth'.format(args.model), net)
net.eval().cuda()
# image and init box
image_files = sorted(glob.glob('../data/bag/*.jpg'))
init_rbox = [334.02, 128.36, 438.19, 188.78, 396.39, 260.83, 292.23, 200.41]
[cx, cy, w, h] = get_axis_aligned_bbox(init_rbox)
# tracker init
target_pos, target_sz = np.array([cx, cy]), np.array([w, h])
im = cv2.imread(image_files[0]) # HxWxC
state = SiamRPN_init(im, target_pos, target_sz, net, args.model)
# tracking and visualization
toc = 0
for f, image_file in enumerate(image_files):
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
dataset_folder = 'imSitu'
imgset_folder = 'resized_256'
train_set = json.load(open(dataset_folder + "/train.json"))
encoder = imsitu_encoder(train_set)
model = model_vsrl_small_finetune.RelationNetworks(encoder, args.gpuid)
# To group up the features
cnn_features, verb_features, role_features = utils.group_features(model)
train_set = imsitu_loader(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=32,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader(imgset_folder, dev_set, encoder,
model.train_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=32,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader(imgset_folder, traindev_set, encoder,
model.train_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
if not args.evaluate:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer(lr, weight_decay, optimizer_select,
cnn_features, verb_features, role_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=lr_step,
gamma=lr_gamma)
#gradient clipping, grad check
if args.evaluate:
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"]
avg_score /= 8
print('Dev average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#write results to csv file
gt_labels = top1.gt_situation
pred_labels = top1.predicted_situation
verb_pred = top1.verb_pred
with open("gt_rn_only.csv", "w") as f:
writer = csv.writer(f)
writer.writerows(gt_labels)
with open("pred_rn_only.csv", "w") as f:
writer = csv.writer(f)
writer.writerows(pred_labels)
with open("verbpred_rn_only.csv", "w") as f:
writer = csv.writer(f)
writer.writerow(['verb', 'total', 'predicted'])
for key, value in verb_pred.items():
writer.writerow([key, value[0], value[1]])
print('Writing predictions to file completed !')
else:
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, optimizer,
scheduler, n_epoch, args.output_dir, encoder, args.gpuid,
clip_norm, lr_max, model_name, args)
def train(iter, epoch_start=0, epoch_stop=100, seed=0):
torch.manual_seed(seed)
net = load_net(iter)
net.train()
cuda = torch.cuda.is_available()
criterion = AlphaLoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[100, 200, 300, 400],
gamma=0.77)
train_set = BoardDataset(load_dataset(iter))
train_loader = DataLoader(train_set,
batch_size=30,
shuffle=True,
num_workers=0,
pin_memory=False)
losses_per_epoch = []
for epoch in tqdm(range(epoch_start, epoch_stop)):
total_loss = 0.0
losses_per_batch = []
for i, data in enumerate(train_loader):
state, policy, value = data
state, policy, value = state.float(), policy.float(), value.float()
if cuda:
state, policy, value = state.cuda(), policy.cuda(), value.cuda(
)
optimizer.zero_grad()
policy_pred, value_pred = net(
state
) # policy_pred = torch.Size([batch, 4 * 9 * 7]) value_pred = torch.Size([batch, 1])
loss = criterion(value_pred[:, 0], value, policy_pred, policy)
loss.backward()
optimizer.step()
total_loss += loss.item()
if i % 10 == 9: # print every 10 mini-batches of size = batch_size
print(
'Process ID: %d [Epoch: %d, %5d/ %d points] total loss per batch: %.3f'
% (os.getpid(), epoch + 1,
(i + 1) * 30, len(train_set), total_loss / 10))
print("Policy:", policy[0].argmax().item(),
policy_pred[0].argmax().item())
print("Value:", value[0].item(), value_pred[0, 0].item())
losses_per_batch.append(total_loss / 10)
total_loss = 0.0
losses_per_epoch.append(sum(losses_per_batch) / len(losses_per_batch))
if len(losses_per_epoch) > 100:
if abs(
sum(losses_per_epoch[-4:-1]) / 3 -
sum(losses_per_epoch[-16:-13]) / 3) <= 0.01:
break
scheduler.step()
fig = plt.figure()
ax = fig.add_subplot()
ax.scatter([e for e in range(1, epoch_stop + 1, 1)], losses_per_epoch)
ax.set_xlabel("Epoch")
ax.set_ylabel("Loss per batch")
ax.set_title("Loss vs Epoch")
print('Finished Training')
plt.savefig(
os.path.join(
"./model_data/", "Loss_vs_Epoch_%s.png" %
datetime.datetime.today().strftime("%Y-%m-%d")))
torch.save(net.state_dict(),
'./model_data/alpha_zero_net_iter{}.pth.tar'.format(iter + 1))
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--role_module',
type=str,
default='',
help='pretrained role module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--dataset_folder',
type=str,
default='./imSitu',
help='Location of annotations')
parser.add_argument('--imgset_dir',
type=str,
default='./resized_256',
help='Location of original images')
parser.add_argument('--frcnn_feat_dir',
type=str,
help='Location of output from detectron')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
#dataset_folder = 'imSitu'
#imgset_folder = 'resized_256'
dataset_folder = args.dataset_folder
imgset_folder = args.imgset_dir
print('model spec :, verb role with context ')
train_set = json.load(open(dataset_folder + "/train.json"))
imsitu_roleq = json.load(open("imsitu_data/imsitu_questions_prev.json"))
encoder = imsitu_encoder(train_set, imsitu_roleq)
model = model_verbmlp_roletd_new.BaseModel(encoder, args.gpuid)
# To group up the features
#all verb and role feat are under role as it's a single unit
train_set = imsitu_loader_roleq(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader_roleq(imgset_folder, dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
test_set = json.load(open(dataset_folder + "/test.json"))
test_set = imsitu_loader_roleq(imgset_folder, test_set, encoder,
model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader_roleq(imgset_folder, traindev_set, encoder,
model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
utils.load_net(args.verb_module, [model.verb])
utils.load_net(args.role_module, [model.roles])
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
torch.manual_seed(1234)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
torch.cuda.manual_seed(1234)
torch.backends.cudnn.deterministic = True
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"]
avg_score /= 3
print('Dev average :{:.2f} {} '.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-')))
import cv2
import face_recognition
import numpy as np
from utils import postprocess, load_class_names, load_colors, load_net, postprocess2
# Fill these
NAME_TO_BLOCK = ''
VIDEO = ''
# Get graph, weight and class files from opencv lib.
GRAPH_PATH = "mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"
WEIGHT_PATH = "frozen_inference_graph.pb"
net = load_net(WEIGHT_PATH, GRAPH_PATH)
classes = load_class_names(path="mscoco_labels.names")
colors = load_colors(path="colors.txt")
#model = mask.create_model()
cap = cv2.VideoCapture(VIDEO)
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print(length)
outputFile = VIDEO[:-4] + '_blocked.avi'
vid_writer = cv2.VideoWriter(outputFile,
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 28,
(round(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))))
# Change these two lines to the person to be blocked
berk_image = face_recognition.load_image_file('known\\berk.jpg')
berk_encoding = face_recognition.face_encodings(berk_image)[0]
path = setup_path(args=args)
setup_logging(path=path, parser=parser)
setup_turi()
gamma = args.gamma
if gamma > 10:
gamma = gamma / 10.0
if gamma > 1.0:
gamma = gamma / 10.0
logging.info(f"Loading networks {gamma}")
networks = list()
for i in range(args.p1, args.p2 + 1):
p = os.path.join(args.nets, f"{gamma}-streamed-{i}.pickle")
g2, net = load_net(p)
networks.append(net)
if g2 != gamma:
logging.warning(
f"Found different gamma in network file {i}: {gamma}!={g2}")
gamma = g2
sizes = [len(net) for net in networks]
logging.info(f"Network sizes: {sizes}")
origin_net = networks[args.origin]
del networks[args.origin]
logging.info(
f"Starting to naive merge {len(networks)} nets with gamma={gamma}")
merged = origin_net.copy()
netD = DeblurDiscriminator().apply(weights_init)
if torch.cuda.is_available():
netG = netG.cuda(opt.gpu)
netD = netD.cuda(opt.gpu)
optim_G = torch.optim.Adam(netG.parameters(),
lr=opt.lr,
betas=(opt.beta1, 0.999))
optim_D = torch.optim.Adam(netD.parameters(),
lr=opt.lr,
betas=(opt.beta1, 0.999))
train(opt, netG, netD, optim_G, optim_D)
if opt.model == 'test':
netG = DeblurGenerator()
load_net(netG, opt.checkpoints_dir, 'G', opt.load_epoch)
if torch.cuda.is_available():
netG = netG.cuda(opt.gpu)
test(opt, netG)
# import matplotlib.pyplot as plt
# a = ReadConcat(opt, transform=image_transform)
# img = a[10]['A']
# print(type(img))
# print(img.shape)
# #img = image_recovery(img)
# img = img.cpu().float().numpy()
# img = (np.transpose(img, (1, 2, 0)) + 1) / 2.0 * 255.0
# img = img.astype(np.uint8)
# plt.imshow(img)
# plt.pause(0)
from utils import parse, load_net, load_data, show_history, show_confusion_matrix
from Model import Model
if __name__ == '__main__':
random.seed(0)
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)
args = parse()
train_loader, test_loader = load_data(args.root, args.csv,
args.input_shape)
net = load_net(args.net, args.pretrained)
if args.load:
net = torch.load(args.load)
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=5e-4)
criterion = nn.CrossEntropyLoss()
model = Model(net, optimizer, criterion)
if args.trainable:
train_history = model.train(train_loader,
epochs=args.epochs,
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
parser.add_argument(
"--command",
choices=["train", "eval", "resume", 'predict', 'finetune'],
required=True)
parser.add_argument("--batch_size", '-b', type=int, default=64)
parser.add_argument("--weights_file", help="the model to start from")
parser.add_argument(
'--finetune_verb',
action='store_true',
help='verb classifier train from the scratch, all others fixed')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
args = parser.parse_args()
batch_size = args.batch_size
#lr = 1e-5
lr = 1e-4
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-5
n_epoch = 500
n_worker = 4
# print('LR scheme : lr decay, vgg, fc as per gnn paper batch 64', 1e-5, 0.1,25)
dataset_folder = 'imSitu'
imgset_folder = 'resized_256'
model_dir = 'trained_models'
train_set = json.load(open(dataset_folder + "/train.json"))
encoder = imsitu_encoder(train_set)
model = model_verb_embd.RelationNetworks(encoder, args.gpuid)
train_set = imsitu_loader(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader(imgset_folder, dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=batch_size,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader(imgset_folder, traindev_set, encoder,
model.train_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=batch_size,
shuffle=True,
num_workers=n_worker)
if args.command == "resume":
print("loading model weights...")
model.load_state_dict(torch.load(args.weights_file))
elif args.finetune_verb:
print(
'CNN fix, Verb fc fixed, train verb classifier layer from the scratch from: {}'
.format(args.verb_module))
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
#print(model)
if args.gpuid >= 0:
print('GPU enabled')
model.cuda()
# optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
utils.set_trainable(model, False)
utils.set_trainable_param(model.classifier.parameters(), True)
utils.set_trainable_param(model.verb.parameters(), True)
optimizer = torch.optim.Adam([{
'params': model.classifier.parameters(),
'lr': 1e-3
}, {
'params': model.verb.parameters(),
'lr': 5e-5
}])
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=lr_step, gamma=lr_gamma)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
'''optimizer = utils.CosineAnnealingWR(0.01,1200000 , 50,
torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))'''
#gradient clipping, grad check
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, optimizer,
scheduler, n_epoch, model_dir, encoder, args.gpuid, clip_norm,
lr_max)
config_init["num_iter_per_epoch"] = 400 * 200 // (config_init["b_source"] + config_init["b_target"])
config_init['nb_class'] = 9 if args.source_dataset in ["CIFAR", "STL"] else 10
config_init['out_path'] = out_dir + out_name + ".txt"
config_init['source_dataset'] = args.source_dataset
config_init['target_dataset'] = args.target_dataset
# log experiment settings:
for key in sorted(config_init):
conf = key + " -> " + str(config_init[key])
log(conf, config_init['out_path'])
# load datasets:
trainloader_source, testloader_source, trainloader_target, testloader_target = load_datasets(config_init)
# load network:
net = load_net(config_init)
# load optimizer:
my_optimizer = torch_optimizer(config_init, net, is_encoder=False)
# to keep initial loss weights (lambda):
config = deepcopy(config_init)
for epoch in range(config["num_epochs"]):
st_time = time.time()
lr, optimizer = my_optimizer.update_lr(epoch)
config = lambda_schedule(config, config_init, epoch)
config = training_loop(net, optimizer, trainloader_source, trainloader_target, config, epoch)
def main(argv):
# folder for saving
subfold = argv[1]
if not os.path.exists(subfold):
os.mkdir(subfold)
# load data
fname = '../LRG-filter/data/lrg_171016/sample-LRG-140-120-10-c3.pkl'
with open(fname, 'rb') as fp:
datadict = pickle.load(fp)
X_raw = datadict['data']
# Reshape and generate train and test dataset
rs = 120
# normalization and whitening
with open("../nets/norm_params.pkl", 'rb') as fp:
normparam = pickle.load(fp)
X_max = normparam["X_max"]
X_min = normparam["X_min"]
X_mean = normparam["X_mean"]
X_train_pre = X_raw
X_train_pre = (X_train_pre - X_min) / (X_max - X_min)
X_in = X_train_pre.reshape(-1, rs, rs, 1)
X_mean = np.mean(X_train_pre)
X_w = X_in - X_mean # Whitening?
numclass = 3
encode_nodes = 32
cnn = ConvNet.ConvNet(input_shape=X_w.shape,
kernel_size=[3, 3, 3, 3, 3],
kernel_num=[8, 8, 16, 32, 32],
fc_nodes=[],
encode_nodes=encode_nodes,
padding=('SAME', 'SAME'),
stride=(2, 2),
numclass=numclass,
sess=None,
name=None)
cnn.cae_build()
cnn.cnn_build(learning_rate=0.001) # In order to init the weights
foldname = "{0}/net_l3_140".format(subfold)
name = "net_l3.pkl"
cnn.sess, cnn.name = utils.load_net(os.path.join(foldname, name))
# estimate
label, label_pos = cnn.cnn_predict(img=X_w)
label_new_pos = pos_to_line(label_pos)
# save result
savefold = "{0}/est_labeled".format(subfold)
if not os.path.exists(savefold):
os.mkdir(savefold)
savepath = "est_l3.pkl"
savedict = {
"label_raw": np.array(datadict['type']),
"z": np.array(datadict['redshift']),
"snvss": np.array(datadict['snvss']),
"name": np.array(datadict['name']),
"label_est": label,
"label_pos": label_new_pos
}
with open(os.path.join(savefold, savepath), 'wb') as fp:
pickle.dump(savedict, fp)
#!/usr/bin/env python3
#print nets stats
import glob
from utils import load_net
if __name__ == "__main__":
for fname in glob.glob('*.pickle'):
try:
g2, net = load_net(fname)
print(f"{fname} gamma={g2} size: {len(net)}")
except:
print(f"Unable to load net from {fname}")
def main():
import argparse
parser = argparse.ArgumentParser(
description="imsitu VSRL. Training, evaluation and prediction.")
parser.add_argument("--gpuid",
default=-1,
help="put GPU id > -1 in GPU mode",
type=int)
#parser.add_argument("--command", choices = ["train", "eval", "resume", 'predict'], required = True)
parser.add_argument('--resume_training',
action='store_true',
help='Resume training from the model [resume_model]')
parser.add_argument('--resume_model',
type=str,
default='',
help='The model we resume')
parser.add_argument('--verb_module',
type=str,
default='',
help='pretrained verb module')
parser.add_argument('--train_role',
action='store_true',
help='cnn fix, verb fix, role train from the scratch')
parser.add_argument(
'--finetune_verb',
action='store_true',
help='cnn fix, verb finetune, role train from the scratch')
parser.add_argument(
'--finetune_cnn',
action='store_true',
help='cnn finetune, verb finetune, role train from the scratch')
parser.add_argument('--output_dir',
type=str,
default='./trained_models',
help='Location to output the model')
parser.add_argument('--evaluate',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--test',
action='store_true',
help='Only use the testing mode')
parser.add_argument('--dataset_folder',
type=str,
default='./imSitu',
help='Location of annotations')
parser.add_argument('--imgset_dir',
type=str,
default='./resized_256',
help='Location of original images')
parser.add_argument('--frcnn_feat_dir',
type=str,
help='Location of output from detectron')
#todo: train role module separately with gt verbs
args = parser.parse_args()
batch_size = 640
#lr = 5e-6
lr = 0.0001
lr_max = 5e-4
lr_gamma = 0.1
lr_step = 25
clip_norm = 50
weight_decay = 1e-4
n_epoch = 500
n_worker = 3
#dataset_folder = 'imSitu'
#imgset_folder = 'resized_256'
dataset_folder = args.dataset_folder
imgset_folder = args.imgset_dir
print('model spec :, top down att with role q ')
train_set = json.load(open(dataset_folder + "/train.json"))
imsitu_roleq = json.load(open("imsitu_data/imsitu_questions.json"))
encoder = imsitu_encoder(train_set, imsitu_roleq)
model = model_role_noun.BaseModel(encoder, args.gpuid)
# To group up the features
cnn_features, role_features = utils.group_features_noun(model)
train_set = imsitu_loader_roleq(imgset_folder, train_set, encoder,
model.train_preprocess())
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
dev_set = json.load(open(dataset_folder + "/dev.json"))
dev_set = imsitu_loader_roleq(imgset_folder, dev_set, encoder,
model.dev_preprocess())
dev_loader = torch.utils.data.DataLoader(dev_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
test_set = json.load(open(dataset_folder + "/test.json"))
test_set = imsitu_loader_roleq(imgset_folder, test_set, encoder,
model.dev_preprocess())
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=64,
shuffle=True,
num_workers=n_worker)
traindev_set = json.load(open(dataset_folder + "/dev.json"))
traindev_set = imsitu_loader_roleq(imgset_folder, traindev_set, encoder,
model.dev_preprocess())
traindev_loader = torch.utils.data.DataLoader(traindev_set,
batch_size=8,
shuffle=True,
num_workers=n_worker)
utils.set_trainable(model, False)
if args.train_role:
print('CNN fix, Verb fix, train role from the scratch from: {}'.format(
args.verb_module))
args.train_all = False
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 1
model_name = 'cfx_vfx_rtrain'
elif args.finetune_verb:
print('CNN fix, Verb finetune, train role from the scratch from: {}'.
format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 2
model_name = 'cfx_vft_rtrain'
elif args.finetune_cnn:
print(
'CNN finetune, Verb finetune, train role from the scratch from: {}'
.format(args.verb_module))
args.train_all = True
if len(args.verb_module) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.verb_module, [model.conv, model.verb],
['conv', 'verb'])
optimizer_select = 3
model_name = 'cft_vft_rtrain'
elif args.resume_training:
print('Resume training from: {}'.format(args.resume_model))
args.train_all = True
if len(args.resume_model) == 0:
raise Exception('[pretrained verb module] not specified')
utils.load_net(args.resume_model, [model])
optimizer_select = 0
model_name = 'resume_all'
else:
print('Training from the scratch.')
optimizer_select = 0
args.train_all = True
model_name = 'train_full'
optimizer = utils.get_optimizer_noun(lr, weight_decay, optimizer_select,
cnn_features, role_features)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
torch.manual_seed(1234)
if args.gpuid >= 0:
#print('GPU enabled')
model.cuda()
torch.cuda.manual_seed(1234)
torch.backends.cudnn.deterministic = True
optimizer = torch.optim.Adamax([{
'params': cnn_features,
'lr': 5e-5
}, {
'params': role_features
}],
lr=1e-3)
#optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=lr_step, gamma=lr_gamma)
#gradient clipping, grad check
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
if args.evaluate:
top1, top5, val_loss = eval(model,
dev_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print('Dev average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#write results to csv file
role_dict = top1.role_dict
fail_val_all = top1.value_all_dict
pass_val_dict = top1.vall_all_correct
fail_role = top1.fail_verb_role
all_role = top1.all_verb_role
with open('role_pred_data.json', 'w') as fp:
json.dump(role_dict, fp, indent=4)
with open('fail_val_all.json', 'w') as fp:
json.dump(fail_val_all, fp, indent=4)
with open('pass_val_all.json', 'w') as fp:
json.dump(pass_val_dict, fp, indent=4)
with open('fail_role.json', 'w') as fp:
json.dump(fail_role, fp, indent=4)
with open('all_role.json', 'w') as fp:
json.dump(all_role, fp, indent=4)
print('Writing predictions to file completed !')
elif args.test:
top1, top5, val_loss = eval(model,
test_loader,
encoder,
args.gpuid,
write_to_file=True)
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print('Test average :{:.2f} {} {}'.format(
avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
else:
print('Model training started!')
train(model, train_loader, dev_loader, traindev_loader, optimizer,
scheduler, n_epoch, args.output_dir, encoder, args.gpuid,
clip_norm, lr_max, model_name, args)
parser.add_argument('--seed', default=0, type=int)
parser.add_argument('--no_cuda', action='store_true', default=False)
args = parser.parse_args()
args.path = get_id(args.data_path)
args.use_cuda = not args.no_cuda and torch.cuda.is_available()
args.device = torch.device('cuda' if args.use_cuda else 'cpu')
torch.manual_seed(args.seed)
# training setup
train_loader, tr_loader_eval, te_loader_eval, num_classes = get_data(
args.dataset, args.data_path, args.bs_train, args.bs_eval,
args.data_size)
crit = nn.CrossEntropyLoss().to(args.device)
net1 = load_net(args.net1_path, num_classes, args.device)
net2 = load_net(args.net2_path, num_classes, args.device)
if args.interp_method == 'string':
points, training_history = get_interpolated_string(
net1, net2, train_loader, args.n_points, args.lr, args.mom,
args.wd, args.stiffness, args.device, crit, args.epochs)
print('Trained the string successfully!')
alphas = np.linspace(0, 1, args.n_points)
tr_evals = []
te_evals = []
for i, alpha in enumerate(alphas):
