def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
init_model()
session_factory = session_factory_from_settings(settings)
if 'localization' not in settings:
settings['localization'] = 'id_ID.UTF-8'
locale.setlocale(locale.LC_ALL, settings['localization'])
if 'timezone' not in settings:
settings['timezone'] = DefaultTimeZone
config = Configurator(settings=settings,
root_factory='reklame.models.RootFactory',
session_factory=session_factory)
config.include('pyramid_beaker')
config.include('pyramid_chameleon')
authn_policy = AuthTktAuthenticationPolicy('sosecret',
callback=group_finder, hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
config.add_request_method(get_user, 'user', reify=True)
config.add_request_method(get_title, 'title', reify=True)
config.add_notfound_view(RemoveSlashNotFoundViewFactory())
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_static_view('deform_static', 'deform:static')
config.add_static_view('files', settings['static_files'])
############################################################################
config.include('pyramid_rpc.jsonrpc') # JSON RPC
json_renderer = JSON()
json_renderer.add_adapter(datetime.datetime, lambda v, request: v.isoformat())
json_renderer.add_adapter(datetime.date, lambda v, request: v.isoformat())
config.add_renderer('myjson', json_renderer)
config.add_jsonrpc_endpoint('ws_reklame', '/ws/reklame', default_renderer="myjson")
############################################################################
routes = DBSession.query(Route.kode, Route.path, Route.nama).all()
for route in routes:
config.add_route(route.kode, route.path)
if route.nama:
titles[route.kode] = ' - '.join([main_title, route.nama])
config.scan()
app = config.make_wsgi_app()
from paste.translogger import TransLogger
app = TransLogger(app, setup_console_handler=False)
return app
def connectToDatabase(self, dburl):
"""
Connect to our database and return a Session object
:param dburl: a valid SQLAlchemy URL string
"""
log.debug("connect db")
engine = db.sa.create_engine(dburl, echo=False)
db.init_model(engine)
session = db.DBSession()
self.setupDatabase(session, engine)
return session
def mixture_experiments(train, dev, glove, splits = 5):
for i in range(splits):
model_name = 'mixture' + str(i)
print 'Model', model_name
model = models.init_model()
div = len(train) / splits
models.train_model(train[:i*div] + train[(i+1)*div:splits*div], dev, glove, model, 'models/' + model_name)
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, "sqlalchemy.")
DBSession.configure(bind=engine)
Base.metadata.bind = engine
init_model()
session_factory = session_factory_from_settings(settings)
if "localization" not in settings:
settings["localization"] = "id_ID.UTF-8"
locale.setlocale(locale.LC_ALL, settings["localization"])
if "timezone" not in settings:
settings["timezone"] = DefaultTimeZone
config = Configurator(
settings=settings, root_factory="inventory.models.RootFactory", session_factory=session_factory
)
config.include("pyramid_beaker")
config.include("pyramid_chameleon")
authn_policy = AuthTktAuthenticationPolicy("sosecret", callback=group_finder, hashalg="sha512")
authz_policy = ACLAuthorizationPolicy()
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
config.add_request_method(get_user, "user", reify=True)
config.add_request_method(get_title, "title", reify=True)
config.add_notfound_view(RemoveSlashNotFoundViewFactory())
config.add_static_view("static", "static", cache_max_age=3600)
config.add_static_view("deform_static", "deform:static")
config.add_static_view("files", settings["static_files"])
routes = DBSession.query(Route.kode, Route.path, Route.nama).all()
for route in routes:
config.add_route(route.kode, route.path)
if route.nama:
titles[route.kode] = " - ".join([main_title, route.nama])
config.scan()
app = config.make_wsgi_app()
from paste.translogger import TransLogger
app = TransLogger(app, setup_console_handler=False)
return app
def extended_tautologies(train, dev, glove):
augment_data = generate_all(train)
from random import shuffle
shuffle(augment_data)
augment_weight = [0, 0.05, 0.15, 0.5]
for w in augment_weight:
new_train = train + augment_data[:int(len(train)*w)]
str = str(w).replace('.','')
model = models.init_model()
models.train_model(new_train, dev, glove, model = model, model_dir = 'models/aug' + w_str)
def grid_experiments(train, dev, glove, embed_size = 300, hidden_size = 100):
lr_vec = [0.001, 0.0003, 0.0001]
dropout_vec = [0.0, 0.1, 0.2]
reg_vec = [0.0, 0.001, 0.0003, 0.0001]
for params in itertools.product(lr_vec, dropout_vec, reg_vec):
filename = 'lr' + str(params[0]).replace('.','') + '_drop' + str(params[1]).replace('.','') + '_reg' + str(params[2]).replace('.','')
print 'Model', filename
model = models.init_model(embed_size, hidden_size, params[0], params[1], params[2])
models.train_model(train, dev, glove, model, 'models/' + filename)
def main():
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
pin_memory = True if use_gpu else False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_img_dataset(
root=args.root,
name=args.dataset,
split_id=args.split_id,
cuhk03_labeled=args.cuhk03_labeled,
cuhk03_classic_split=args.cuhk03_classic_split,
)
# data augmentation
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
trainloader = DataLoader(
ImageDataset(dataset.train, transform=transform_train),
sampler=RandomIdentitySampler(dataset.train,
num_instances=args.num_instances),
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
queryloader = DataLoader(
ImageDataset(dataset.query, transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
galleryloader = DataLoader(
ImageDataset(dataset.gallery, transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch,
num_classes=dataset.num_train_pids,
loss={'softmax', 'metric'},
aligned=True,
use_gpu=use_gpu)
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
if args.labelsmooth:
criterion_class = CrossEntropyLabelSmooth(
num_classes=dataset.num_train_pids, use_gpu=use_gpu)
else:
criterion_class = CrossEntropyLoss(use_gpu=use_gpu)
criterion_metric = TripletLossAlignedReID(margin=args.margin)
optimizer = init_optim(args.optim, model.parameters(), args.lr,
args.weight_decay)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.stepsize,
gamma=args.gamma)
start_epoch = args.start_epoch
if args.resume:
print("Loading checkpoint from '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, use_gpu)
return 0
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion_class, criterion_metric, optimizer,
trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
if args.stepsize > 0: scheduler.step()
if (epoch + 1) > args.start_eval and args.eval_step > 0 and (
epoch + 1) % args.eval_step == 0 or (epoch +
1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
)
galleryloader = DataLoader(
VideoDataset(dataset.gallery,
seq_len=args.seq_len,
sample='dense',
transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
model = models.init_model(name=args.arch,
num_classes=dataset.num_train_pids,
loss={'xent', 'htri'})
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion_htri = TripletLoss(margin, 'cosine')
criterion_xent = CrossEntropyLabelSmooth(dataset.num_train_pids)
criterion_center_loss = CenterLoss(use_gpu=use_gpu)
if args.use_OSMCAA:
print("USING OSM LOSS")
print("config, alpha = %f sigma = %f l=%f" % (alpha, sigma, l))
criterion_osm_caa = OSM_CAA_Loss(alpha=alpha, l=l, osm_sigma=sigma)
else:
criterion_osm_caa = None
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'), mode='a')
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_dataset(name=args.dataset, root=args.root)
# Data augmentation
spatial_transform_train = [
ST.Scale((args.height, args.width), interpolation=3),
ST.RandomHorizontalFlip(),
ST.ToTensor(),
ST.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]
spatial_transform_train = ST.Compose(spatial_transform_train)
temporal_transform_train = TT.TemporalRandomCrop(size=args.seq_len,
stride=args.sample_stride)
spatial_transform_test = ST.Compose([
ST.Scale((args.height, args.width), interpolation=3),
ST.ToTensor(),
ST.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
temporal_transform_test = TT.TemporalBeginCrop(size=args.test_frames)
pin_memory = True if use_gpu else False
dataset_train = dataset.train
if args.dataset == 'duke':
dataset_train = dataset.train_dense
print('process duke dataset')
trainloader = DataLoader(
VideoDataset(dataset_train,
spatial_transform=spatial_transform_train,
temporal_transform=temporal_transform_train),
sampler=RandomIdentitySampler(dataset_train,
num_instances=args.num_instances),
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
queryloader = DataLoader(VideoDataset(
dataset.query,
spatial_transform=spatial_transform_test,
temporal_transform=temporal_transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False)
galleryloader = DataLoader(VideoDataset(
dataset.gallery,
spatial_transform=spatial_transform_test,
temporal_transform=temporal_transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch,
use_gpu=use_gpu,
num_classes=dataset.num_train_pids,
loss={'xent', 'htri'})
print(model)
criterion_xent = nn.CrossEntropyLoss()
criterion_htri = TripletLoss(margin=args.margin,
distance=args.distance,
use_gpu=use_gpu)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=args.stepsize,
gamma=args.gamma)
start_epoch = args.start_epoch
if use_gpu:
model = nn.DataParallel(model).cuda()
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion_xent, criterion_htri, optimizer,
trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
if (epoch + 1) >= args.start_eval and (
epoch + 1) % args.eval_step == 0 or epoch == 0:
print("==> Test")
with torch.no_grad():
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
if 'pbb.url' in settings and settings['pbb.url']:
from models.pbb import (
pbbDBSession,
pbbBase)
pbb_engine = engine_from_config(settings, 'pbb.')
pbbDBSession.configure(bind=pbb_engine)
pbbBase.metadata.bind = pbb_engine
pbbBase.pbb_schema = 'pbb_schema' in settings and settings['pbb_schema'] or 'PBB'
init_model()
session_factory = session_factory_from_settings(settings)
if 'localization' not in settings:
settings['localization'] = 'id_ID.UTF-8'
locale.setlocale(locale.LC_ALL, settings['localization'])
if 'timezone' not in settings:
settings['timezone'] = DefaultTimeZone
config = Configurator(settings=settings,
root_factory='opensipkdpbb.models.RootFactory',
session_factory=session_factory)
config.include('pyramid_beaker')
config.include('pyramid_chameleon')
authn_policy = AuthTktAuthenticationPolicy('sosecret',
callback=group_finder, hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
config.add_request_method(get_user, 'user', reify=True)
config.add_request_method(get_title, 'title', reify=True)
config.add_notfound_view(RemoveSlashNotFoundViewFactory())
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_static_view('deform_static', 'deform:static')
config.add_static_view('files', settings['static_files'])
############################################################################
from pyramid.renderers import JSON
import datetime
config.include('pyramid_rpc.jsonrpc') # JSON RPC
json_renderer = JSON()
json_renderer.add_adapter(datetime.datetime, lambda v, request: v.isoformat())
json_renderer.add_adapter(datetime.date, lambda v, request: v.isoformat())
config.add_renderer('myjson', json_renderer)
config.add_jsonrpc_endpoint('ws_user', '/ws/user', default_renderer="myjson")
config.add_jsonrpc_endpoint('ws_pbb', '/ws/pbb', default_renderer="myjson")
############################################################################
for name, path, title in routes:
config.add_route(name, path)
config.scan()
app = config.make_wsgi_app()
from paste.translogger import TransLogger
app = TransLogger(app, setup_console_handler=False)
return app
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_dataset(
name=args.dataset,
split_id=args.split_id,
cuhk03_labeled=args.cuhk03_labeled,
cuhk03_classic_split=args.cuhk03_classic_split,
)
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
#arch1 should use salience, and arch2 should use semantic parsing
use_salience = models.use_salience(name=args.arch1)
use_parsing = models.use_parsing(name=args.arch2)
save_rank = True if args.save_rank else False
use_re_ranking = True if args.use_re_ranking else False
queryloader = DataLoader(
ImageDataset(dataset.query,
transform=transform_test,
use_salience=use_salience,
use_parsing=use_parsing,
salience_base_path=dataset.salience_query_dir,
parsing_base_path=dataset.parsing_query_dir),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
galleryloader = DataLoader(
ImageDataset(dataset.gallery,
transform=transform_test,
use_salience=use_salience,
use_parsing=use_parsing,
salience_base_path=dataset.salience_gallery_dir,
parsing_base_path=dataset.parsing_gallery_dir),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
print("Initializing model: {}".format(args.arch1))
model1 = models.init_model(name=args.arch1,
num_classes=dataset.num_train_pids,
loss={'xent', 'htri'},
mid_layer=args.mid_layer)
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model1.parameters()) / 1000000.0))
print("Initializing model: {}".format(args.arch2))
model2 = models.init_model(name=args.arch2,
num_classes=dataset.num_train_pids,
loss={'xent', 'htri'},
mid_layer=args.mid_layer)
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model2.parameters()) / 1000000.0))
print("Loading checkpoint from '{}'".format(args.resume1))
checkpoint = torch.load(args.resume1)
model1.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
print("Resuming model 1 from epoch {}".format(start_epoch + 1))
print("Loading checkpoint from '{}'".format(args.resume2))
checkpoint = torch.load(args.resume2)
model2.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
print("Resuming model 2 from epoch {}".format(start_epoch + 1))
if use_gpu:
model1 = nn.DataParallel(model1).cuda()
model2 = nn.DataParallel(model2).cuda()
test(model1,
model2,
queryloader,
galleryloader,
use_gpu,
use_salience=use_salience,
use_parsing=use_parsing,
save_dir=args.save_dir,
epoch=-1,
save_rank=save_rank,
use_re_ranking=use_re_ranking)
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_vidreid_dataset(root=args.root, name=args.dataset)
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
# decompose tracklets into images for image-based training
new_train = []
for img_paths, pid, camid in dataset.train:
for img_path in img_paths:
new_train.append((img_path, pid, camid))
trainloader = DataLoader(
ImageDataset(new_train, transform=transform_train),
sampler=RandomIdentitySampler(new_train, num_instances=args.num_instances),
batch_size=args.train_batch, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
queryloader = DataLoader(
VideoDataset(dataset.query, seq_len=args.seq_len, sample='evenly', transform=transform_test),
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
galleryloader = DataLoader(
VideoDataset(dataset.gallery, seq_len=args.seq_len, sample='evenly', transform=transform_test),
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch, num_classes=dataset.num_train_pids, loss={'xent', 'htri'})
print("Model size: {:.3f} M".format(count_num_param(model)))
criterion_xent = CrossEntropyLabelSmooth(num_classes=dataset.num_train_pids, use_gpu=use_gpu)
criterion_htri = TripletLoss(margin=args.margin)
optimizer = init_optim(args.optim, model.parameters(), args.lr, args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=args.stepsize, gamma=args.gamma)
start_epoch = args.start_epoch
if args.load_weights:
# load pretrained weights but ignore layers that don't match in size
print("Loading pretrained weights from '{}'".format(args.load_weights))
checkpoint = torch.load(args.load_weights)
pretrain_dict = checkpoint['state_dict']
model_dict = model.state_dict()
pretrain_dict = {k: v for k, v in pretrain_dict.items() if k in model_dict and model_dict[k].size() == v.size()}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
if args.resume:
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
rank1 = checkpoint['rank1']
print("Loaded checkpoint from '{}'".format(args.resume))
print("- start_epoch: {}\n- rank1: {}".format(start_epoch, rank1))
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, args.pool, use_gpu)
return
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion_xent, criterion_htri, optimizer, trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
if (epoch+1) > args.start_eval and args.eval_step > 0 and (epoch+1) % args.eval_step == 0 or (epoch+1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, args.pool, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best, osp.join(args.save_dir, 'checkpoint_ep' + str(epoch+1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print("Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".format(elapsed, train_time))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_dataset(name=args.dataset)
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
# decompose tracklets into images for image-based training
new_train = []
for img_paths, pid, camid in dataset.train:
for img_path in img_paths:
new_train.append((img_path, pid, camid))
trainloader = DataLoader(
ImageDataset(new_train, transform=transform_train),
sampler=RandomIdentitySampler(new_train,
num_instances=args.num_instances),
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
queryloader = DataLoader(
VideoDataset(dataset.query,
seq_len=args.seq_len,
sample='evenly',
transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
galleryloader = DataLoader(
VideoDataset(dataset.gallery,
seq_len=args.seq_len,
sample='evenly',
transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch,
num_classes=dataset.num_train_pids,
loss={'xent', 'htri'})
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion_xent = CrossEntropyLabelSmooth(
num_classes=dataset.num_train_pids, use_gpu=use_gpu)
criterion_htri = TripletLoss(margin=args.margin)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.stepsize,
gamma=args.gamma)
start_epoch = args.start_epoch
if args.resume:
print("Loading checkpoint from '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, args.pool, use_gpu)
return
start_time = time.time()
best_rank1 = -np.inf
for epoch in range(start_epoch, args.max_epoch):
print("==> Epoch {}/{}".format(epoch + 1, args.max_epoch))
train(model, criterion_xent, criterion_htri, optimizer, trainloader,
use_gpu)
if args.stepsize > 0: scheduler.step()
if args.eval_step > 0 and (epoch + 1) % args.eval_step == 0 or (
epoch + 1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, args.pool, use_gpu)
is_best = rank1 > best_rank1
if is_best: best_rank1 = rank1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_vidreid_dataset(root=args.root,
name=args.dataset)
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
# decompose tracklets into images
new_train = []
for img_paths, pid, camid in dataset.train:
for img_path in img_paths:
new_train.append((img_path, pid, camid))
trainloader = DataLoader(
ImageDataset(new_train, transform=transform_train),
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
queryloader = DataLoader(
VideoDataset(dataset.query,
seq_len=args.seq_len,
sample='evenly',
transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
galleryloader = DataLoader(
VideoDataset(dataset.gallery,
seq_len=args.seq_len,
sample='evenly',
transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch,
num_classes=dataset.num_train_pids,
loss={'xent'})
print("Model size: {:.3f} M".format(count_num_param(model)))
criterion = CrossEntropyLabelSmooth(num_classes=dataset.num_train_pids,
use_gpu=use_gpu)
optimizer = init_optim(args.optim, model.parameters(), args.lr,
args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=args.stepsize,
gamma=args.gamma)
if args.fixbase_epoch > 0:
if hasattr(model, 'classifier') and isinstance(model.classifier,
nn.Module):
optimizer_tmp = init_optim(args.optim,
model.classifier.parameters(),
args.fixbase_lr, args.weight_decay)
else:
print(
"Warn: model has no attribute 'classifier' and fixbase_epoch is reset to 0"
)
args.fixbase_epoch = 0
if args.load_weights:
# load pretrained weights but ignore layers that don't match in size
print("Loading pretrained weights from '{}'".format(args.load_weights))
checkpoint = torch.load(args.load_weights)
pretrain_dict = checkpoint['state_dict']
model_dict = model.state_dict()
pretrain_dict = {
k: v
for k, v in pretrain_dict.items()
if k in model_dict and model_dict[k].size() == v.size()
}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
if args.resume:
if osp.isfile(args.resume):
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
args.start_epoch = checkpoint['epoch']
rank1 = checkpoint['rank1']
print("Loaded checkpoint from '{}'".format(args.resume))
print("- start_epoch: {}\n- rank1: {}".format(
args.start_epoch, rank1))
else:
print("=> No checkpoint found at '{}'".format(args.resume))
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
distmat = test(model,
queryloader,
galleryloader,
use_gpu,
return_distmat=True)
if args.vis_ranked_res:
visualize_ranked_results(
distmat,
dataset,
save_dir=osp.join(args.save_dir, 'ranked_results'),
topk=20,
)
return
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
if args.fixbase_epoch > 0:
print(
"Train classifier for {} epochs while keeping base network frozen".
format(args.fixbase_epoch))
for epoch in range(args.fixbase_epoch):
start_train_time = time.time()
train(epoch,
model,
criterion,
optimizer_tmp,
trainloader,
use_gpu,
freeze_bn=True)
train_time += round(time.time() - start_train_time)
del optimizer_tmp
print("Now open all layers for training")
for epoch in range(args.start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion, optimizer, trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
if (epoch + 1) > args.start_eval and args.eval_step > 0 and (
epoch + 1) % args.eval_step == 0 or (epoch +
1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, args.pool, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def main():
#GENERAL
torch.cuda.empty_cache()
root = "/home/kuru/Desktop/veri-gms-master_noise/"
train_dir = '/home/kuru/Desktop/veri-gms-master_noise/VeRispan/image_train/'
source = {'verispan'}
target = {'verispan'}
workers = 4
height = 240
width = 240
train_size = 32
train_sampler = 'RandomSampler'
#AUGMENTATION
random_erase = True
jitter = True
aug = True
#OPTIMIZATION
opt = 'adam'
lr = 0.0003
weight_decay = 5e-4
momentum = 0.9
sgd_damp = 0.0
nesterov = True
warmup_factor = 0.01
warmup_method = 'linear'
#HYPERPARAMETER
max_epoch = 80
start = 0
train_batch_size = 16
test_batch_size = 50
#SCHEDULER
lr_scheduler = 'multi_step'
stepsize = [30, 60]
gamma = 0.1
#LOSS
margin = 0.3
num_instances = 4
lambda_tri = 1
#MODEL
#arch = 'resnet101'
arch = 'resnet101_ibn_a'
no_pretrained = False
#TEST SETTINGS
load_weights = '/home/kuru/Desktop/veri-gms-master/IBN-Net_pytorch0.4.1/resnet101_ibn_a.pth'
#load_weights = None
start_eval = 0
eval_freq = -1
#MISC
use_gpu = True
print_freq = 10
seed = 1
resume = ''
save_dir = '/home/kuru/Desktop/veri-gms-master_noise/spanningtree_verinoise_101_stride2/'
gpu_id = 0, 1
vis_rank = True
query_remove = True
evaluate = False
dataset_kwargs = {
'source_names': source,
'target_names': target,
'root': root,
'height': height,
'width': width,
'train_batch_size': train_batch_size,
'test_batch_size': test_batch_size,
'train_sampler': train_sampler,
'random_erase': random_erase,
'color_jitter': jitter,
'color_aug': aug
}
transform_kwargs = {
'height': height,
'width': width,
'random_erase': random_erase,
'color_jitter': jitter,
'color_aug': aug
}
optimizer_kwargs = {
'optim': opt,
'lr': lr,
'weight_decay': weight_decay,
'momentum': momentum,
'sgd_dampening': sgd_damp,
'sgd_nesterov': nesterov
}
lr_scheduler_kwargs = {
'lr_scheduler': lr_scheduler,
'stepsize': stepsize,
'gamma': gamma
}
use_gpu = torch.cuda.is_available()
log_name = 'log_test.txt' if evaluate else 'log_train.txt'
sys.stdout = Logger(osp.join(save_dir, log_name))
print('Currently using GPU ', gpu_id)
cudnn.benchmark = True
print('Initializing image data manager')
#dataset = init_imgreid_dataset(root='/home/kuru/Desktop/veri-gms-master/', name='veri')
dataset = init_imgreid_dataset(
root='/home/kuru/Desktop/veri-gms-master_noise/', name='verispan')
train = []
num_train_pids = 0
num_train_cams = 0
print(len(dataset.train))
for img_path, pid, camid, subid, countid in dataset.train:
#print(img_path)
path = img_path[56 + 6:90 + 6]
#print(path)
folder = path[1:4]
#print(folder)
#print(img_path, pid, camid,subid,countid)
pid += num_train_pids
camid += num_train_cams
newidd = 0
train.append((path, folder, pid, camid, subid, countid))
#print(train)
#break
num_train_pids += dataset.num_train_pids
num_train_cams += dataset.num_train_cams
pid = 0
pidx = {}
for img_path, pid, camid, subid, countid in dataset.train:
path = img_path[56 + 6:90 + 6]
folder = path[1:4]
pidx[folder] = pid
pid += 1
#print(pidx)
sub = []
final = 0
xx = dataset.train
newids = []
print(train[0:2])
train2 = {}
for k in range(0, 770):
for img_path, pid, camid, subid, countid in dataset.train:
if k == pid:
newid = final + subid
sub.append(newid)
#print(pid,subid,newid)
newids.append(newid)
train2[img_path] = newid
#print(img_path, pid, camid, subid, countid, newid)
final = max(sub)
#print(final)
print(len(newids), final)
#train=train2
#print(train2)
train3 = []
for img_path, pid, camid, subid, countid in dataset.train:
#print(img_path,pid,train2[img_path])
path = img_path[56 + 6:90 + 6]
#print(path)
folder = path[1:4]
newid = train2[img_path]
#print((path, folder, pid, camid, subid, countid,newid ))
train3.append((path, folder, pid, camid, subid, countid, newid))
train = train3
# for (path, folder, pid, camid, subid, countid,newid) in train:
# print(path, folder)
path = '/home/kuru/Desktop/adhi/veri-final-draft-master_noise/gmsNoise776/'
pkl = {}
#pkl[0] = pickle.load('/home/kuru/Desktop/veri-gms-master/gms/620.pkl')
entries = os.listdir(path)
for name in entries:
f = open((path + name), 'rb')
ccc = (path + name)
#print(ccc)
if name == 'featureMatrix.pkl':
s = name[0:13]
else:
s = name[0:3]
#print(s)
#with open (ccc,"rb") as ff:
# pkl[s] = pickle.load(ff)
#print(pkl[s])
pkl[s] = pickle.load(f)
f.close
#print(len(pkl))
print('=> pickle indexing')
data_index = search(pkl)
print(len(data_index))
# with open('cids.pkl', 'rb') as handle:
# b = pickle.load(handle)
# #print(b)
# with open('index.pkl', 'rb') as handle:
# c = pickle.load(handle)
transform_t = train_transforms(**transform_kwargs)
#print(train[0],train[10])
# train4=[]
# for path, folder, pid, camid, subid, countid,newid in train:
# if countid > 3:
# train4.append((path, folder, pid, camid, subid, countid,newid ))
# print(len(train4))
# train=train4
#data_tfr = vd(pkl_file='index.pkl', dataset = train, root_dir='/home/kuru/Desktop/veri-gms-master/VeRi/image_train/', transform=transform_t)
data_tfr = vdspan(
pkl_file='index_veryspan_noise.pkl',
dataset=train,
root_dir=
'/home/kuru/Desktop/veri-gms-master_noise/VeRispan/image_train/',
transform=transform_t)
#print(data_tfr)
#print(trainloader)
#data_tfr2=list(data_tfr)
df2 = []
data_tfr_old = data_tfr
for (img, label, index, pid, cid, subid, countid, newid) in data_tfr:
#print("datframe",(label))
#print(countid)
if countid > 4:
#print(countid)
df2.append((img, label, index, pid, cid, subid, countid, newid))
print("filtered final trainset length", len(df2))
data_tfr = df2
# with open('df2noise_ex.pkl', 'wb') as handle:
# b = pickle.dump(df2, handle, protocol=pickle.HIGHEST_PROTOCOL)
# with open('df2noise.pkl', 'rb') as handle:
# df2 = pickle.load(handle)
# data_tfr=df2
# for (img,label,index,pid, cid,subid,countid,newid) in data_tfr :
# print("datframe",(label))
#data_tfr = vdspansort( dataset = train, root_dir='/home/kuru/Desktop/veri-gms-master_noise/VeRispan/image_train/', transform=transform_t)
#trainloader = DataLoader(df2, sampler=None,batch_size=train_batch_size, shuffle=True, num_workers=workers,pin_memory=True, drop_last=True)
trainloader = DataLoader(data_tfr,
sampler=None,
batch_size=train_batch_size,
shuffle=True,
num_workers=workers,
pin_memory=True,
drop_last=True)
for batch_idx, (img, label, index, pid, cid, subid, countid,
newid) in enumerate(trainloader):
#print("trainloader",batch_idx, (label,index,pid, cid,subid,countid,newid))
print("trainloader", batch_idx, (label))
break
print('Initializing test data manager')
dm = ImageDataManager(use_gpu, **dataset_kwargs)
testloader_dict = dm.return_dataloaders()
print('Initializing model: {}'.format(arch))
model = models.init_model(name=arch,
num_classes=num_train_pids,
loss={'xent', 'htri'},
pretrained=not no_pretrained,
last_stride=2)
print('Model size: {:.3f} M'.format(count_num_param(model)))
if load_weights is not None:
print("weights loaded")
load_pretrained_weights(model, load_weights)
#checkpoint = torch.load('/home/kuru/Desktop/veri-gms-master/logg/model.pth.tar-19')
#model._load_from_state_dict(checkpoint['state_dict'])
#model.load_state_dict(checkpoint['state_dict'])
#optimizer.load_state_dict(checkpoint['optimizer'])
#print(checkpoint['epoch'])
#print(checkpoint['rank1'])
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
print(torch.cuda.device_count())
model = nn.DataParallel(model).cuda() if use_gpu else model
optimizer = init_optimizer(model, **optimizer_kwargs)
#optimizer = init_optimizer(model)
#optimizer.load_state_dict(checkpoint['optimizer'])
scheduler = init_lr_scheduler(optimizer, **lr_scheduler_kwargs)
# scheduler = WarmupMultiStepLR(optimizer, STEPS, GAMMA,
# WARMUP_FACTOR,
# WARMUP_EPOCHS, WARMUP_METHOD)
criterion_xent = CrossEntropyLoss(num_classes=num_train_pids,
use_gpu=use_gpu,
label_smooth=True)
criterion_htri = TripletLoss(margin=margin)
ranking_loss = nn.MarginRankingLoss(margin=margin)
if evaluate:
print('Evaluate only')
for name in target:
print('Evaluating {} ...'.format(name))
queryloader = testloader_dict[name]['query']
galleryloader = testloader_dict[name]['gallery']
_, distmat = test(model,
queryloader,
galleryloader,
train_batch_size,
use_gpu,
return_distmat=True)
if vis_rank:
visualize_ranked_results(distmat,
dm.return_testdataset_by_name(name),
save_dir=osp.join(
save_dir, 'ranked_results', name),
topk=20)
return
time_start = time.time()
ranklogger = RankLogger(source, target)
# # checkpoint = torch.load('/home/kuru/Desktop/market_all/ibna_model/model.pth.tar-79')
# # model.load_state_dict(checkpoint['state_dict'])
# # optimizer.load_state_dict(checkpoint['optimizer'])
# # print(checkpoint['epoch'])
# # start_epoch=checkpoint['epoch']
# # start=start_epoch
# checkpoint = torch.load('/home/kuru/Desktop/veri-gms-master/spanningtreeveri/model.pth.tar-2')
# model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
# print(checkpoint['epoch'])
# start_epoch=checkpoint['epoch']
# start=start_epoch
##start_epoch=resume_from_checkpoint('/home/kuru/Desktop/veri-gms-master/logg/model.pth.tar-20', model, optimizer=None)
print('=> Start training')
for epoch in range(start, max_epoch):
print(epoch, scheduler.get_lr()[0])
#print( torch.cuda.memory_allocated(0))
losses = AverageMeter()
#xent_losses = AverageMeter()
htri_losses = AverageMeter()
accs = AverageMeter()
batch_time = AverageMeter()
xent_losses = AverageMeter()
model.train()
for p in model.parameters():
p.requires_grad = True # open all layers
end = time.time()
for batch_idx, (img, label, index, pid, cid, subid, countid,
newid) in enumerate(trainloader):
trainX, trainY = torch.zeros(
(train_batch_size * 3, 3, height, width),
dtype=torch.float32), torch.zeros((train_batch_size * 3),
dtype=torch.int64)
#pids = torch.zeros((batch_size*3), dtype = torch.int16)
#batchcount=0
for i in range(train_batch_size):
if (countid[i] > 4):
#batchcount=batchcount+1
#print("dfdsfs")
labelx = label[i]
indexx = index[i]
cidx = pid[i]
if indexx > len(pkl[labelx]) - 1:
indexx = len(pkl[labelx]) - 1
#maxx = np.argmax(pkl[labelx][indexx])
a = pkl[labelx][indexx]
minpos = np.argmin(ma.masked_where(a == 0, a))
# print(len(a))
# print(a)
# print(ma.masked_where(a==0, a))
# print(labelx,index,pid,cidx,minpos)
# print(np.array(data_index).shape)
# print(data_index[cidx][1])
pos_dic = data_tfr_old[data_index[cidx][1] + minpos]
#print('posdic', pos_dic)
neg_label = int(labelx)
while True:
neg_label = random.choice(range(1, 770))
#print(neg_label)
if neg_label is not int(labelx) and os.path.isdir(
os.path.join(
'/home/kuru/Desktop/veri-gms-master_noise/veriNoise_train_spanning_folder',
strint(neg_label))) is True:
break
negative_label = strint(neg_label)
neg_cid = pidx[negative_label]
neg_index = random.choice(
range(0, len(pkl[negative_label])))
#print(negative_label,neg_cid,neg_index,data_index[neg_cid] )
neg_dic = data_tfr_old[data_index[neg_cid][1] + neg_index]
#print('negdic', neg_dic)
trainX[i] = img[i]
trainX[i + train_batch_size] = pos_dic[0]
trainX[i + (train_batch_size * 2)] = neg_dic[0]
trainY[i] = cidx
trainY[i + train_batch_size] = pos_dic[3]
trainY[i + (train_batch_size * 2)] = neg_dic[3]
# trainY[i+train_batch_size] = pos_dic[7]
# trainY[i+(train_batch_size*2)] = neg_dic[7]
#break
# else:
# print("skiped",countid[i],subid[i],label[i])
#break
#print(batchcount)
trainX = trainX.cuda()
trainY = trainY.cuda()
outputs, features = model(trainX)
xent_loss = criterion_xent(outputs[0:train_batch_size],
trainY[0:train_batch_size])
htri_loss = criterion_htri(features, trainY)
# centerloss= CENTER_LOSS_WEIGHT * center_criterion(features, trainY)
#tri_loss = ranking_loss(features)
#ent_loss = xent_loss(outputs[0:batch_size], trainY[0:batch_size], num_train_pids)
#loss = htri_loss+xent_loss + centerloss
loss = htri_loss + xent_loss
optimizer.zero_grad()
#optimizer_center.zero_grad()
loss.backward()
optimizer.step()
# for param in center_criterion.parameters():
# param.grad.data *= (1. /CENTER_LOSS_WEIGHT)
# optimizer_center.step()
for param_group in optimizer.param_groups:
#print(param_group['lr'] )
lrrr = str(param_group['lr'])
batch_time.update(time.time() - end)
losses.update(loss.item(), trainY.size(0))
htri_losses.update(htri_loss.item(), trainY.size(0))
xent_losses.update(xent_loss.item(), trainY.size(0))
accs.update(
accuracy(outputs[0:train_batch_size],
trainY[0:train_batch_size])[0])
if (batch_idx) % 50 == 0:
print('Train ', end=" ")
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'TriLoss {loss.val:.4f} ({loss.avg:.4f})\t'
'XLoss {xloss.val:.4f} ({xloss.avg:.4f})\t'
'OveralLoss {oloss.val:.4f} ({oloss.avg:.4f})\t'
'Acc {acc.val:.2f} ({acc.avg:.2f})\t'
'lr {lrrr} \t'.format(
epoch + 1,
batch_idx + 1,
len(trainloader),
batch_time=batch_time,
loss=htri_losses,
xloss=xent_losses,
oloss=losses,
acc=accs,
lrrr=lrrr,
))
end = time.time()
# del loss
# del htri_loss
# del xent_loss
# del htri_losses
# del losses
# del outputs
# del features
# del accs
# del trainX
# del trainY
scheduler.step()
print('=> Test')
save_checkpoint(
{
'state_dict': model.state_dict(),
#'rank1': rank1,
'epoch': epoch + 1,
'arch': arch,
'optimizer': optimizer.state_dict(),
},
save_dir)
GPUtil.showUtilization()
print(torch.cuda.memory_allocated(), torch.cuda.memory_cached())
for name in target:
print('Evaluating {} ...'.format(name))
queryloader = testloader_dict[name]['query']
galleryloader = testloader_dict[name]['gallery']
rank1, distmat = test(model, queryloader, galleryloader,
test_batch_size, use_gpu)
ranklogger.write(name, epoch + 1, rank1)
rank2, distmat2 = test_rerank(model, queryloader, galleryloader,
test_batch_size, use_gpu)
ranklogger.write(name, epoch + 1, rank2)
del queryloader
del galleryloader
del distmat
print(torch.cuda.memory_allocated(), torch.cuda.memory_cached())
torch.cuda.empty_cache()
if (epoch + 1) == max_epoch:
#if (epoch + 1) % 10 == 0:
print('=> Test')
save_checkpoint(
{
'state_dict': model.state_dict(),
'rank1': rank1,
'epoch': epoch + 1,
'arch': arch,
'optimizer': optimizer.state_dict(),
}, save_dir)
for name in target:
print('Evaluating {} ...'.format(name))
queryloader = testloader_dict[name]['query']
galleryloader = testloader_dict[name]['gallery']
rank1, distmat = test(model, queryloader, galleryloader,
test_batch_size, use_gpu)
ranklogger.write(name, epoch + 1, rank1)
# del queryloader
# del galleryloader
# del distmat
if vis_rank:
visualize_ranked_results(
distmat,
dm.return_testdataset_by_name(name),
save_dir=osp.join(save_dir, 'ranked_results', name),
topk=20)
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_img_dataset(
root=args.root,
name=args.dataset,
split_id=args.split_id,
cuhk03_labeled=args.cuhk03_labeled,
cuhk03_classic_split=args.cuhk03_classic_split,
)
transform_train = T.Compose([
T.Resize((args.height, args.width)),
T.RandomHorizontalFlip(p=0.5),
T.Pad(10),
T.RandomCrop([args.height, args.width]),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
torchvision.transforms.RandomErasing(p=0.5,
scale=(0.02, 0.4),
ratio=(0.3, 3.33),
value=(0.4914, 0.4822, 0.4465))
#T.RandomErasing(probability=0.5, sh=0.4, mean=(0.4914, 0.4822, 0.4465)),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
trainloader = DataLoader(
ImageDataset(dataset.train, transform=transform_train),
batch_size=args.train_batch,
shuffle=True,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
queryloader = DataLoader(
ImageDataset(dataset.query, transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
galleryloader = DataLoader(
ImageDataset(dataset.gallery, transform=transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch,
num_classes=dataset.num_train_pids,
loss={'ring'},
use_gpu=use_gpu)
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
criterion_xent = CrossEntropyLabelSmooth(
num_classes=dataset.num_train_pids, use_gpu=use_gpu)
criterion_ring = RingLoss(args.weight_ring)
if use_gpu: criterion_ring = criterion_ring.cuda()
params = list(model.parameters()) + list(criterion_ring.parameters())
optimizer = init_optim(args.optim, params, args.lr, args.weight_decay)
# if args.stepsize > 0:
# scheduler = lr_scheduler.StepLR(optimizer, step_size=args.stepsize, gamma=args.gamma)
'''------Modify lr_schedule here------'''
current_schedule = init_lr_schedule(schedule=args.schedule,
warm_up_epoch=args.warm_up_epoch,
half_cos_period=args.half_cos_period,
lr_milestone=args.lr_milestone,
gamma=args.gamma,
stepsize=args.stepsize)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
lr_lambda=current_schedule)
'''------Please refer to the args.xxx for details of hyperparams------'''
# embed()
start_epoch = args.start_epoch
if args.resume:
print("Loading checkpoint from '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, use_gpu)
return
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion_xent, criterion_ring, optimizer,
trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
if args.schedule: scheduler.step()
if (epoch + 1) > args.start_eval and args.eval_step > 0 and (
epoch + 1) % args.eval_step == 0 or (epoch +
1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def map_permutation_test(args):
# Set up logger
logger = getLogger(args.verbosity)
# Load required modules
from sklearn.model_selection import LeaveOneOut, GridSearchCV, cross_val_predict
from metrics import compute_metrics
# Load the input data
X = pd.read_csv(args.feature_file, index_col=0, sep='\t')
y = pd.read_csv(args.outcome_file, index_col=0, sep='\t')
feature_classes = pd.read_csv(args.feature_class_file, index_col=0, sep='\t')
# Align the features and outcomes
patients = X.index
X = X.reindex(index = patients)
y = y.reindex(index = patients)
outcome_name = y.columns[0]
# Restrict to the training columns
selected_feature_classes = set(map(str.capitalize, set(FEATURE_CLASSES) - set(args.excluded_feature_classes)))
training_cols = feature_classes['Class'].isin(selected_feature_classes).index.tolist()
############################################################################
# RUN PERMUTATION TEST
############################################################################
#Initialize model
pipeline, gscv = init_model(args.model, args.n_jobs,
args.estimator_random_seed, args.max_iter, args.tol)
# Permute the outcomes
np.random.seed(args.permutation_random_seed)
y[outcome_name] = np.random.permutation(y[outcome_name])
# Convert dataframes to matrices to avoid dataframe splitting error
outer_cv = LeaveOneOut()
preds = pd.Series(cross_val_predict(estimator = gscv,
X=X.loc[:,training_cols],
y=y[outcome_name], cv=outer_cv,
n_jobs = args.n_jobs,
verbose=61 if args.verbosity > 0 else 0),
index = patients)
# Evalue results
sub_y = y.loc[patients][outcome_name].values
sub_preds = preds.loc[patients].values
metric_vals, var_explained = compute_metrics(sub_y, sub_preds)
############################################################################
# OUTPUT TO FILE
############################################################################
with open(args.output_file, 'w') as OUT:
output = {
"var_explained": var_explained.tolist(),
"true": sub_y.tolist(),
"preds": "sub_preds",
"params": vars(args),
"training_features": training_cols
}
output.update(metric_vals.items())
json.dump( output, OUT )
def init_app(app):
from models import init_model
init_model(app)
def main():
runId = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
cfg.OUTPUT_DIR = os.path.join(cfg.OUTPUT_DIR, runId)
if not os.path.exists(cfg.OUTPUT_DIR):
os.mkdir(cfg.OUTPUT_DIR)
print(cfg.OUTPUT_DIR)
torch.manual_seed(cfg.RANDOM_SEED)
random.seed(cfg.RANDOM_SEED)
np.random.seed(cfg.RANDOM_SEED)
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.MODEL.DEVICE_ID
use_gpu = torch.cuda.is_available() and cfg.MODEL.DEVICE == "cuda"
if not cfg.EVALUATE_ONLY:
sys.stdout = Logger(osp.join(cfg.OUTPUT_DIR, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(cfg.OUTPUT_DIR, 'log_test.txt'))
print("==========\nConfigs:{}\n==========".format(cfg))
if use_gpu:
print("Currently using GPU {}".format(cfg.MODEL.DEVICE_ID))
cudnn.benchmark = True
torch.cuda.manual_seed_all(cfg.RANDOM_SEED)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(cfg.DATASETS.NAME))
dataset = data_manager.init_dataset(root=cfg.DATASETS.ROOT_DIR,
name=cfg.DATASETS.NAME)
if cfg.ATTR_RECOG_ON:
cfg.DATASETS.ATTR_LENS = dataset.attr_lens
else:
cfg.DATASETS.ATTR_LENS = []
cfg.DATASETS.ATTR_COLUMNS = dataset.columns
print("Initializing model: {}".format(cfg.MODEL.NAME))
if cfg.MODEL.ARCH == 'video_baseline':
torch.backends.cudnn.benchmark = False
model = models.init_model(name=cfg.MODEL.ARCH,
num_classes=dataset.num_train_pids,
pretrain_choice=cfg.MODEL.PRETRAIN_CHOICE,
last_stride=cfg.MODEL.LAST_STRIDE,
neck=cfg.MODEL.NECK,
model_name=cfg.MODEL.NAME,
neck_feat=cfg.TEST.NECK_FEAT,
model_path=cfg.MODEL.PRETRAIN_PATH,
fusion_method=cfg.MODEL.FUSION_METHOD,
attr_lens=cfg.DATASETS.ATTR_LENS,
attr_loss=cfg.DATASETS.ATTRIBUTE_LOSS)
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
transform_train = T.Compose([
T.Resize(cfg.INPUT.SIZE_TRAIN),
T.RandomHorizontalFlip(p=cfg.INPUT.PROB),
T.Pad(cfg.INPUT.PADDING),
T.RandomCrop(cfg.INPUT.SIZE_TRAIN),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
T.RandomErasing(probability=cfg.INPUT.RE_PROB,
mean=cfg.INPUT.PIXEL_MEAN)
])
transform_test = T.Compose([
T.Resize(cfg.INPUT.SIZE_TEST),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
# pin_memory = True if use_gpu else False
pin_memory = False
cfg.DATALOADER.NUM_WORKERS = 0
cfg.DATASETS.ATTR_COLUMNS = dataset.columns
trainloader = DataLoader(VideoDataset(
dataset.train,
seq_len=cfg.DATASETS.SEQ_LEN,
sample=cfg.DATASETS.TRAIN_SAMPLE_METHOD,
transform=transform_train,
attr_loss=cfg.DATASETS.ATTRIBUTE_LOSS,
attr_lens=cfg.DATASETS.ATTR_LENS,
dataset_name=cfg.DATASETS.NAME),
sampler=RandomIdentitySampler(
dataset.train,
num_instances=cfg.DATALOADER.NUM_INSTANCE),
batch_size=cfg.SOLVER.SEQS_PER_BATCH,
num_workers=cfg.DATALOADER.NUM_WORKERS,
pin_memory=pin_memory,
drop_last=True)
queryloader = DataLoader(VideoDataset(
dataset.query,
seq_len=cfg.DATASETS.SEQ_LEN,
sample=cfg.DATASETS.TEST_SAMPLE_METHOD,
transform=transform_test,
max_seq_len=cfg.DATASETS.TEST_MAX_SEQ_NUM,
dataset_name=cfg.DATASETS.NAME,
attr_lens=cfg.DATASETS.ATTR_LENS,
attr_loss=cfg.DATASETS.ATTRIBUTE_LOSS),
batch_size=cfg.TEST.SEQS_PER_BATCH,
shuffle=False,
num_workers=cfg.DATALOADER.NUM_WORKERS,
pin_memory=pin_memory,
drop_last=False)
galleryloader = DataLoader(
VideoDataset(dataset.gallery,
seq_len=cfg.DATASETS.SEQ_LEN,
sample=cfg.DATASETS.TEST_SAMPLE_METHOD,
transform=transform_test,
max_seq_len=cfg.DATASETS.TEST_MAX_SEQ_NUM,
dataset_name=cfg.DATASETS.NAME,
attr_lens=cfg.DATASETS.ATTR_LENS,
attr_loss=cfg.DATASETS.ATTRIBUTE_LOSS),
batch_size=cfg.TEST.SEQS_PER_BATCH,
shuffle=False,
num_workers=cfg.DATALOADER.NUM_WORKERS,
pin_memory=pin_memory,
drop_last=False,
)
if cfg.MODEL.SYN_BN:
model = apex.parallel.convert_syncbn_model(model)
optimizer = make_optimizer(cfg, model)
if cfg.SOLVER.FP_16:
model, optimizer = apex.amp.initialize(model.cuda(),
optimizer,
opt_level='O1')
if use_gpu:
model = nn.DataParallel(model)
model.cuda()
start_time = time.time()
xent = CrossEntropyLabelSmooth(num_classes=dataset.num_train_pids)
if cfg.DISTANCE == "cosine":
tent = CosineTripletLoss(cfg.SOLVER.MARGIN)
elif cfg.DISTANCE == "euclid":
tent = TripletLoss(cfg.SOLVER.MARGIN)
if cfg.DATASETS.ATTRIBUTE_LOSS == "mce":
attr_criter = nn.CrossEntropyLoss()
elif cfg.DATASETS.ATTRIBUTE_LOSS == "bce":
attr_criter = nn.BCEWithLogitsLoss()
tlaw = TripletLossAttrWeightes(dis_type=cfg.DISTANCE)
scheduler = WarmupMultiStepLR(optimizer, cfg.SOLVER.STEPS,
cfg.SOLVER.GAMMA, cfg.SOLVER.WARMUP_FACTOR,
cfg.SOLVER.WARMUP_ITERS,
cfg.SOLVER.WARMUP_METHOD)
ema = None
no_rise = 0
metrics = test(model, queryloader, galleryloader,
cfg.TEST.TEMPORAL_POOL_METHOD, use_gpu)
# return
best_rank1 = 0
start_epoch = 0
for epoch in range(start_epoch, cfg.SOLVER.MAX_EPOCHS):
# if no_rise == 10:
# break
scheduler.step()
print("noriase:", no_rise)
print("==> Epoch {}/{}".format(epoch + 1, cfg.SOLVER.MAX_EPOCHS))
print("current lr:", scheduler.get_lr()[0])
train(model,
trainloader,
xent,
tent,
attr_criter,
optimizer,
use_gpu,
tlaw=tlaw)
if cfg.SOLVER.EVAL_PERIOD > 0 and (
(epoch + 1) % cfg.SOLVER.EVAL_PERIOD == 0 or
(epoch + 1) == cfg.SOLVER.MAX_EPOCHS):
print("==> Test")
metrics = test(model, queryloader, galleryloader,
cfg.TEST.TEMPORAL_POOL_METHOD, use_gpu)
rank1 = metrics[0]
if rank1 > best_rank1:
best_rank1 = rank1
no_rise = 0
else:
no_rise += 1
continue
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
torch.save(
state_dict,
osp.join(
cfg.OUTPUT_DIR, "rank1_" + str(rank1) + '_checkpoint_ep' +
str(epoch + 1) + '.pth'))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
def main():
global args, best_prec1, use_gpu
args = parser.parse_args()
use_gpu = torch.cuda.is_available()
num_classes = 79
# define state params
state = {'batch_size': args.batch_size, 'image_size': args.image_size, 'max_epochs': args.epochs,
'evaluate': args.evaluate, 'resume': args.resume, 'num_classes':num_classes, 'load': args.load, 'test': args.test}
state['difficult_examples'] = True
state['save_model_path'] = args.checkpoint
state['workers'] = args.workers
state['epoch_step'] = args.epoch_step
state['lr'] = args.lr
state['device_ids'] = args.device_ids
if args.evaluate:
state['evaluate'] = True
if args.test:
state['test'] = True
if not args.test:
#TODO: Make annotation paths more general
train_dataset = DatasetLoader(args.data, img_set='train', annotation=os.path.join('/srv/data1/ashishsingh/Half_and_Half_Data/I2L/annotation', 'antisymm_multilabel_nofreq_trainset_annotation.json'))
if args.val_hnh:
val_dataset = DatasetLoader_HNH(args.data, img_set='val', annotation=os.path.join('/srv/data1/ashishsingh/Half_and_Half_Data/I2L/annotation', 'i2l_valset_annotation.json'))
else:
val_dataset = DatasetLoader(args.data, img_set='val_coco', annotation=os.path.join('/srv/data1/ashishsingh/Half_and_Half_Data', 'valset_complete_metadata.json'))
print("Initializing model: {}".format(args.arch))
if args.pretrained:
model = models.init_model(name=args.arch, num_classes=num_classes, pretrained = 'imagenet', use_gpu=use_gpu)
else:
model = models.init_model(name=args.arch, num_classes=num_classes, pretrained = None, use_gpu=use_gpu)
print("Model size: {:.3f} M".format(count_num_param(model)))
# define loss function (criterion)
#criterion = nn.MultiLabelSoftMarginLoss()
criterion = MultiLabelSoftmaxLoss()
if args.val_hnh or args.test:
criterion_val = nn.CrossEntropyLoss()
else:
criterion_val = nn.MultiLabelSoftMarginLoss()
# define optimizer
optimizer = torch.optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
#Validate using test-like objective
if args.val_hnh:
engine = SymmetricMultiLabelHNHEngine(state)
else:
engine = SymmetricMultiLabelMAPEngine(state)
engine.learning(model, criterion, criterion_val, train_dataset, val_dataset, optimizer)
else:
test_dataset = DatasetLoader_HNH(args.data, img_set='test_cleaned', annotation=os.path.join('/srv/data1/ashishsingh/Half_and_Half_Data/I2L/annotation', 'i2l_testset_annotation.json'))
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch, num_classes=num_classes, pretrained = None, use_gpu=use_gpu)
criterion_test = nn.CrossEntropyLoss()
engine = SymmetricMultiLabelHNHEngine(state)
engine.test(model, criterion_test, test_dataset)
def main():
global args
args = parser.parse_args()
# Set the GPU device IDs
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpuid
cudnn.benchmark = True
# Create folder to save prediction images
save_root = args.dout
if not osp.isdir(save_root):
os.makedirs(save_root)
# Initialize model and display total parameters
model = models.init_model(name='dfusenet')
print("Total parameters: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
# Load pre-trained weights
if os.path.isfile(args.model):
print("Loading pre-trained weights '{}'".format(args.model))
checkpoint = torch.load(args.model)
model.load_state_dict(checkpoint['state_dict'])
else:
print(
"No model found, please check location and try again '{}'".format(
args.resume))
return
model = torch.nn.DataParallel(model).cuda()
tensor_to_pil = T.ToPILImage()
image_folder = sorted(glob.glob(args.rgbin + '/*.png'))
laser_folder = sorted(glob.glob(args.din + '/*.png'))
model.eval()
print("Beginning evaluation..")
with torch.no_grad():
for img_idx in range(0, len(laser_folder)):
# Iterate through input folder contents
laser_path = laser_folder[img_idx]
left_path = image_folder[img_idx]
print("Currently processing the following images..")
print("RGB Image: {}".format(left_path))
print("D Image: {}".format(laser_path))
# Load and pre-process input lidar data
laser_pil = Image.open(laser_path)
laser = in_depth_transform(laser_pil)
# Load and pre-process input RGB data
left_pil = Image.open(left_path)
left = color_transform(left_pil)
# Convert RGB and lidar data to Tensor
left = TF.to_tensor(left).float()
laser = TF.to_tensor(laser).float()
laser = torch.unsqueeze(laser, 0).cuda()
left = torch.unsqueeze(left, 0).cuda()
# We stack the input and rgb tensors as one tensor for simplicity
input_stack = torch.cat((left, laser), dim=1)
# Perform forward pass through the network
output = model(input_stack)
# Rescale data back into original KITTI format
output = output * 256.
laser = laser * 256.
output = output[0].cpu()
pil_img = tensor_to_pil(output.int())
filename = laser_folder[img_idx].split('/')[-1]
pil_img.save(save_root + filename)
print('Finished processing: {}'.format(filename))
def main():
# 是否使用GPU和是否节省显存
use_gpu = torch.cuda.is_available()
if args.use_cpu:
use_gpu = False
pin_memory = True if use_gpu else False
# Log文件的输出
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("============\nArgs:{}\n=============".format(args))
# GPU调用
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
cudnn.benchmark = True # 表示使用cudnn
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU !")
# 初始化dataset
dataset = data_manager.Market1501(root=args.root)
# dataloader(train query gallery) 和 增强
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(p=0.5),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
transform_test = T.Compose([
T.Resize(args.height, args.width),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
trainloader = DataLoader(
ImageDataset(dataset.train, transform=transform_train),
sampler=RandomIdentitySampler(dataset.train,
num_instance=args.num_instances),
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=
True # 训练时会存在epoch % batchsize != 0 的情况，那么余数的图片是否还要训练？drop_last= True就是舍去这些图片
)
queryloader = DataLoader(ImageDataset(dataset.query,
transform=transform_test),
batch_size=args.train_batch,
num_workers=args.workers,
shuffle=False,
pin_memory=pin_memory)
galleryloader = DataLoader(ImageDataset(dataset.gallery,
transform=transform_test),
batch_size=args.train_batch,
num_workers=args.workers,
shuffle=False,
pin_memory=pin_memory)
# 加载模型
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch,
num_classes=dataset.num_train_pids,
loss={'softmax', 'metric'})
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
# 损失和优化器
criterion_class = nn.CrossEntropyLoss()
criterion_metric = TripletLoss(margin=args.margin)
# optimizer = torch.optim.adam()
# 只更新其中某两层(先运行下行语句看看要更新哪几层)
# print(*list(model.children()))
# optimizer = init_optim(args.optim, model.parameters(nn.Sequential([
# *list(model.children())[:-2]
# ])))
optimizer = init_optim(args.optim, model.parameters(), args.lr,
args.weight_decay)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.stepsize,
gamma=args.gamma)
start_epoch = args.start_epoch
# 是否要恢复模型
if args.resume:
print("Loading checkpoint from {}".format(args.resume))
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
# 并行
if use_gpu:
model = nn.DataParallel(model).cuda()
# 如果只是想测试
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, use_gpu)
return 0
start_time = time.time()
train_time = 0
# 训练
print("Start Traing!")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion_class, criterion_metric, optimizer,
trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
# 测试以及存模型
# step()了才会衰减
if args.stepsize > 0:
scheduler.step()
if (epoch + 1) > args.start_eval and args.eval_step > 0 and (
epoch + 1) % args.eval_step == 0 or (epoch +
1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
# set a learning rate
if args.lr_factor == -1:
args.lr_factor = random()
args.lr = args.lr_factor * 10**-args.lr_base
args.lr *= len(args.gpu_devices.split(','))
print(f"Choose learning rate {args.lr}")
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'), mode='a')
print("==========\nArgs:{}\n==========".format(args))
#assert torch.distributed.is_available()
#print("Initializing DDP by nccl-tcp({}) rank({}) world_size({})".format(args.init_method, args.rank, args.world_size))
#dist.init_process_group(backend='nccl', init_method=args.init_method, rank=args.rank, world_size=args.world_size)
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_dataset(name=args.dataset, root=args.root)
# Data augmentation
spatial_transform_train = [
ST.Scale((args.height, args.width), interpolation=3),
ST.RandomHorizontalFlip(),
ST.ToTensor(),
ST.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
ST.RandomErasing(probability=0.5, mean=[0.485, 0.456, 0.406])
]
spatial_transform_train = ST.Compose(spatial_transform_train)
temporal_transform_train = TT.TemporalRandomCrop(size=args.seq_len,
stride=args.sample_stride)
#temporal_transform_train = TT.TemporalRandomCropPick(size=args.seq_len, stride=args.sample_stride)
spatial_transform_test = ST.Compose([
ST.Scale((args.height, args.width), interpolation=3),
ST.ToTensor(),
ST.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
temporal_transform_test = TT.TemporalBeginCrop(size=args.test_frames)
pin_memory = True if use_gpu else False
dataset_train = dataset.train
if args.dataset == 'duke':
dataset_train = dataset.train_dense
print('process duke dataset')
#sampler = RandomIdentitySampler(dataset_train, num_instances=args.num_instances)
if args.dataset == 'lsvid':
sampler = RandomIdentityCameraSampler(dataset_train,
num_instances=args.num_instances,
num_cam=dataset.num_camids)
elif args.dataset == 'mars':
sampler = RandomIdentityCameraSampler(dataset_train,
num_instances=args.num_instances,
num_cam=dataset.num_camids)
trainloader = DataLoader(
VideoDataset(dataset_train,
spatial_transform=spatial_transform_train,
temporal_transform=temporal_transform_train),
sampler=sampler,
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
'''
for batch_idx, (vids, pids, camids, img_paths) in enumerate(trainloader):
print(batch_idx, pids, camids, img_paths)
break
return
'''
dataset_query = dataset.query
dataset_gallery = dataset.gallery
if args.dataset == 'lsvid':
dataset_query = dataset.val_query
dataset_gallery = dataset.val_gallery
print('process lsvid dataset')
queryloader = DataLoader(VideoDataset(
dataset_query,
spatial_transform=spatial_transform_test,
temporal_transform=temporal_transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False)
galleryloader = DataLoader(VideoDataset(
dataset_gallery,
spatial_transform=spatial_transform_test,
temporal_transform=temporal_transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False)
print("Initializing model: {}".format(args.arch))
model = models.init_model(
name=args.arch,
use_gpu=use_gpu,
num_classes=[dataset.num_train_pids, dataset.num_camids],
loss={'xent', 'htri'},
vis=True)
#print(model)
if args.resume:
print("Loading checkpoint from '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
criterion_xent = nn.CrossEntropyLoss()
criterion_htri = TripletLoss(margin=args.margin,
distance=args.distance,
use_gpu=use_gpu)
criterion_htri_c = TripletInterCamLoss(margin=args.margin,
distance=args.distance,
use_gpu=use_gpu)
criterion_attn = AttentionLoss()
#criterion_htri_c = TripletWeightedInterCamLoss(margin=args.margin, distance=args.distance, use_gpu=use_gpu, alpha=args.cam_alpha)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
#scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=args.stepsize, gamma=args.gamma)
scheduler = WarmupMultiStepLR(optimizer,
milestones=args.stepsize,
gamma=args.gamma,
warmup_factor=1.0 / 10,
warmup_iters=10,
warmup_method="linear")
start_epoch = args.start_epoch
if use_gpu:
model = nn.DataParallel(model).cuda()
#model = model.cuda()
#model = nn.parallel.DistributedDataParallel(model)
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
#print("Set sampler seed to {}".format(args.seed*epoch))
#sampler.set_seed(args.seed*epoch)
if args.resume and epoch + 1 <= args.resume_epoch:
print(f"Skip epoch {epoch+1}")
scheduler.step()
continue
start_train_time = time.time()
train(epoch, model, criterion_xent, criterion_htri, criterion_htri_c,
criterion_attn, optimizer, trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
if (epoch + 1) >= args.start_eval and (
epoch + 1) % args.eval_step == 0 or epoch == 0:
print("==> Test")
with torch.no_grad():
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint(
{
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_dataset(name=args.dataset, root=args.root)
# Data augmentation
spatial_transform_train = ST.Compose([
ST.Scale((args.height, args.width), interpolation=3),
ST.RandomHorizontalFlip(),
ST.ToTensor(),
ST.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
temporal_transform_train = TT.TemporalRandomCrop(size=args.seq_len,
stride=args.sample_stride)
spatial_transform_test = ST.Compose([
ST.Scale((args.height, args.width), interpolation=3),
ST.ToTensor(),
ST.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
temporal_transform_test = TT.TemporalBeginCrop()
transform_test_img = T.Compose([
T.Resize((args.height, args.width), interpolation=3),
T.ToTensor(),
T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
pin_memory = True if use_gpu else False
if args.dataset == 'dukevid':
trainloader = DataLoader(
VideoDataset(dataset.train_dense,
spatial_transform=spatial_transform_train,
temporal_transform=temporal_transform_train),
sampler=RandomIdentitySampler(dataset.train_dense,
num_instances=args.num_instances),
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
else:
trainloader = DataLoader(
VideoDataset(dataset.train,
spatial_transform=spatial_transform_train,
temporal_transform=temporal_transform_train),
sampler=RandomIdentitySampler(dataset.train,
num_instances=args.num_instances),
batch_size=args.train_batch,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=True,
)
queryloader = DataLoader(VideoDataset(
dataset.query,
spatial_transform=spatial_transform_test,
temporal_transform=temporal_transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=0,
pin_memory=pin_memory,
drop_last=False)
galleryloader = DataLoader(VideoDataset(
dataset.gallery,
spatial_transform=spatial_transform_test,
temporal_transform=temporal_transform_test),
batch_size=args.test_batch,
shuffle=False,
num_workers=0,
pin_memory=pin_memory,
drop_last=False)
queryimgloader = DataLoader(ImageDataset(dataset.query_img,
transform=transform_test_img),
batch_size=args.img_test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False)
galleryimgloader = DataLoader(ImageDataset(dataset.gallery_img,
transform=transform_test_img),
batch_size=args.img_test_batch,
shuffle=False,
num_workers=args.workers,
pin_memory=pin_memory,
drop_last=False)
print("Initializing model: {} and {}".format(args.vid_arch, args.img_arch))
vid_model = models.init_model(name=args.vid_arch)
img_model = models.init_model(name=args.img_arch)
classifier = models.init_model(name='classifier',
num_classes=dataset.num_train_pids)
print("Video model size: {:.5f}M".format(
sum(p.numel() for p in vid_model.parameters()) / 1000000.0))
print("Image model size: {:.5f}M".format(
sum(p.numel() for p in img_model.parameters()) / 1000000.0))
criterion = nn.CrossEntropyLoss()
criterion_tkp_f = FeatureBasedTKP(bp_to_vid=args.bp_to_vid)
criterion_tkp_d = SimilarityBasedTKP(distance='euclidean',
bp_to_vid=args.bp_to_vid)
criterion_i2v = HeterogeneousTripletLoss(margin=0.3, distance='euclidean')
optimizer = torch.optim.Adam([{
'params': vid_model.parameters(),
'lr': args.lr
}, {
'params': img_model.parameters(),
'lr': args.lr
}, {
'params': classifier.parameters(),
'lr': args.lr
}],
weight_decay=args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=args.stepsize,
gamma=args.gamma)
start_epoch = args.start_epoch
if args.resume:
print("Loading checkpoint from '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
vid_model.load_state_dict(checkpoint['vid_model_state_dict'])
img_model.load_state_dict(checkpoint['img_model_state_dict'])
classifier.load_state_dict(checkpoint['classifier_state_dict'])
start_epoch = checkpoint['epoch']
if use_gpu:
vid_model = vid_model.cuda()
img_model = img_model.cuda()
classifier = classifier.cuda()
if args.evaluate:
print("Evaluate only")
with torch.no_grad():
test(vid_model, img_model, queryloader, galleryloader,
queryimgloader, galleryimgloader, use_gpu)
return
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
scheduler.step()
print('1')
start_train_time = time.time()
print('2')
train(epoch, vid_model, img_model, classifier, criterion,
criterion_tkp_f, criterion_tkp_d, criterion_i2v, optimizer,
trainloader, use_gpu)
#torch.cuda.empty_cache()
train_time += round(time.time() - start_train_time)
if (epoch + 1) >= args.start_eval and args.eval_step > 0 and (
epoch + 1) % args.eval_step == 0 or (epoch +
1) == args.max_epoch:
print("==> Test")
with torch.no_grad():
rank1 = test(vid_model, img_model, queryloader, galleryloader,
queryimgloader, galleryimgloader, use_gpu)
torch.cuda.empty_cache()
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
vid_model_state_dict = vid_model.state_dict()
img_model_state_dict = img_model.state_dict()
classifier_state_dict = classifier.state_dict()
save_checkpoint(
{
'vid_model_state_dict': vid_model_state_dict,
'img_model_state_dict': img_model_state_dict,
'classifier_state_dict': classifier_state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best,
osp.join(args.save_dir,
'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(
best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def init_app( app):
from models import init_model
init_model( app)
from __future__ import print_function, absolute_import
import argparse
import models
parser = argparse.ArgumentParser(
description='Train image model with cross entropy loss')
# Architecture
parser.add_argument('-a',
'--arch',
type=str,
default='resnet50',
choices=models.get_names())
args = parser.parse_args()
if __name__ == '__main__':
model = models.init_model(name=args.arch)
print("Model size: {:.5f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
parser.add_argument('--profile')
parser.add_argument('--render', action="store_false", default=False)
# runner params
parser.add_argument('--num_episodes', type=int, default=10000000)
parser.add_argument('--num_runners', type=int, default=1)
# MCTS params
parser.add_argument('--mcts_iters', type=int, default=50)
parser.add_argument('--mcts_c_puct', type=float, default=1.0)
# RL params
parser.add_argument('--discount', type=float, default=0.9)
parser.add_argument('--temperature', type=float, default=0.4)
args = parser.parse_args()
assert args.num_episodes % args.num_runners == 0, "The number of episodes should be divisible by number of runners"
if LOAD_TRAINED_MODEL: model = load_model(args.model_file)
else: model = models.init_model()
# # use spawn method for starting subprocesses
# ctx = multiprocessing.get_context('spawn')
#
# # create fifos and processes for all runners
# fifo = ctx.Queue()
# for i in range(args.num_runners):
# process = ctx.Process(target=profile_runner if args.profile else train, args=(i, args.num_episodes // args.num_runners, fifo, args, model))
# process.start()
train(args.num_episodes, args, model)
# # do logging in the main process
# all_rewards = []
# all_lengths = []
def main(opt):
if not osp.exists(save_dir): os.makedirs(save_dir)
if not osp.exists(vis_dir): os.makedirs(vis_dir)
use_gpu = torch.cuda.is_available()
pin_memory = True if use_gpu else False
if args.mode == 'train':
sys.stdout = Logger(osp.join(save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("GPU mode")
cudnn.benchmark = True
torch.cuda.manual_seed(args.seed)
else:
print("CPU mode")
### Setup dataset loader ###
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_img_dataset(
root=args.root,
name=args.dataset,
split_id=opt['split_id'],
cuhk03_labeled=opt['cuhk03_labeled'],
cuhk03_classic_split=opt['cuhk03_classic_split'])
if args.ak_type < 0:
trainloader = DataLoader(
ImageDataset(dataset.train, transform=opt['transform_train']),
sampler=RandomIdentitySampler(dataset.train,
num_instances=opt['num_instances']),
batch_size=args.train_batch,
num_workers=opt['workers'],
pin_memory=pin_memory,
drop_last=True)
elif args.ak_type > 0:
trainloader = DataLoader(ImageDataset(
dataset.train, transform=opt['transform_train']),
sampler=AttrPool(dataset.train,
args.dataset,
attr_matrix,
attr_list,
sample_num=16),
batch_size=args.train_batch,
num_workers=opt['workers'],
pin_memory=pin_memory,
drop_last=True)
queryloader = DataLoader(ImageDataset(dataset.query,
transform=opt['transform_test']),
batch_size=args.test_batch,
shuffle=False,
num_workers=opt['workers'],
pin_memory=pin_memory,
drop_last=False)
galleryloader = DataLoader(ImageDataset(dataset.gallery,
transform=opt['transform_test']),
batch_size=args.test_batch,
shuffle=False,
num_workers=opt['workers'],
pin_memory=pin_memory,
drop_last=False)
### Prepare criterion ###
if args.ak_type < 0:
clf_criterion = adv_CrossEntropyLabelSmooth(
num_classes=dataset.num_train_pids,
use_gpu=use_gpu) if args.loss in ['xent', 'xent_htri'
] else adv_CrossEntropyLoss(
use_gpu=use_gpu)
else:
clf_criterion = nn.MultiLabelSoftMarginLoss()
metric_criterion = adv_TripletLoss(margin=args.margin,
ak_type=args.ak_type)
criterionGAN = GANLoss()
### Prepare pretrained model ###
target_net = models.init_model(name=args.targetmodel,
pre_dir=pre_dir,
num_classes=dataset.num_train_pids)
check_freezen(target_net, need_modified=True, after_modified=False)
### Prepare main net ###
G = Generator(3, 3, args.num_ker,
norm=args.normalization).apply(weights_init)
if args.D == 'PatchGAN':
D = Pat_Discriminator(input_nc=6,
norm=args.normalization).apply(weights_init)
elif args.D == 'MSGAN':
D = MS_Discriminator(input_nc=6,
norm=args.normalization,
temperature=args.temperature,
use_gumbel=args.usegumbel).apply(weights_init)
check_freezen(G, need_modified=True, after_modified=True)
check_freezen(D, need_modified=True, after_modified=True)
print("Model size: {:.5f}M".format(
(sum(g.numel()
for g in G.parameters()) + sum(d.numel()
for d in D.parameters())) /
1000000.0))
# setup optimizer
optimizer_G = optim.Adam(G.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999))
optimizer_D = optim.Adam(D.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999))
if use_gpu:
test_target_net = nn.DataParallel(target_net).cuda(
) if not args.targetmodel == 'pcb' else nn.DataParallel(
PCB_test(target_net)).cuda()
target_net = nn.DataParallel(target_net).cuda()
G = nn.DataParallel(G).cuda()
D = nn.DataParallel(D).cuda()
if args.mode == 'test':
epoch = 'test'
test(G,
D,
test_target_net,
dataset,
queryloader,
galleryloader,
epoch,
use_gpu,
is_test=True)
return 0
# Ready
start_time = time.time()
train_time = 0
worst_mAP, worst_rank1, worst_rank5, worst_rank10, worst_epoch = np.inf, np.inf, np.inf, np.inf, 0
best_hit, best_epoch = -np.inf, 0
print("==> Start training")
for epoch in range(1, args.epoch + 1):
start_train_time = time.time()
train(epoch, G, D, target_net, criterionGAN, clf_criterion,
metric_criterion, optimizer_G, optimizer_D, trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
if epoch % args.eval_freq == 0:
print("==> Eval at epoch {}".format(epoch))
if args.ak_type < 0:
cmc, mAP = test(G,
D,
test_target_net,
dataset,
queryloader,
galleryloader,
epoch,
use_gpu,
is_test=False)
is_worst = cmc[0] <= worst_rank1 and cmc[
1] <= worst_rank5 and cmc[
2] <= worst_rank10 and mAP <= worst_mAP
if is_worst:
worst_mAP, worst_rank1, worst_epoch = mAP, cmc[0], epoch
print(
"==> Worst_epoch is {}, Worst mAP {:.1%}, Worst rank-1 {:.1%}"
.format(worst_epoch, worst_mAP, worst_rank1))
save_checkpoint(
G.state_dict(), is_worst, 'G',
osp.join(save_dir, 'G_ep' + str(epoch) + '.pth.tar'))
save_checkpoint(
D.state_dict(), is_worst, 'D',
osp.join(save_dir, 'D_ep' + str(epoch) + '.pth.tar'))
else:
all_hits = test(G,
D,
target_net,
dataset,
queryloader,
galleryloader,
epoch,
use_gpu,
is_test=False)
is_best = all_hits[0] >= best_hit
if is_best:
best_hit, best_epoch = all_hits[0], epoch
print("==> Best_epoch is {}, Best rank-1 {:.1%}".format(
best_epoch, best_hit))
save_checkpoint(
G.state_dict(), is_best, 'G',
osp.join(save_dir, 'G_ep' + str(epoch) + '.pth.tar'))
save_checkpoint(
D.state_dict(), is_best, 'D',
osp.join(save_dir, 'D_ep' + str(epoch) + '.pth.tar'))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".
format(elapsed, train_time))
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.bind = engine
init_model()
###Engine Tambahan
if settings['pbb.url']:
from models.pbb import (
PbbDBSession,
PbbBase
)
os.environ["NLS_LANG"] = ".AL32UTF8"
pbb_engine = engine_from_config(settings, 'pbb.') #, encoding='.AL32UTF8', convert_unicode=True)
PbbDBSession.configure(bind=pbb_engine)
PbbBase.metadata.bind = pbb_engine
pbb_schema = settings['pbb_schema']
if settings['pospbb.url']:
from models.pospbb import (
PosPbbDBSession,
PosPbbBase,
)
pospbb_engine = engine_from_config(settings, 'pospbb.')
PosPbbDBSession.configure(bind=pospbb_engine)
PosPbbBase.metadata.bind = pospbb_engine
pospbb_schema = settings['pospbb_schema']
session_factory = session_factory_from_settings(settings)
if 'localization' not in settings:
settings['localization'] = 'id_ID.UTF-8'
locale.setlocale(locale.LC_ALL, settings['localization'])
if 'timezone' not in settings:
settings['timezone'] = DefaultTimeZone
config = Configurator(settings=settings,
root_factory='pbbrekon.models.RootFactory',
session_factory=session_factory)
config.include('pyramid_beaker')
config.include('pyramid_chameleon')
authn_policy = AuthTktAuthenticationPolicy('sosecret',
callback=group_finder, hashalg='sha512')
authz_policy = ACLAuthorizationPolicy()
config.set_authentication_policy(authn_policy)
config.set_authorization_policy(authz_policy)
config.add_request_method(get_user, 'user', reify=True)
config.add_request_method(get_title, 'title', reify=True)
config.add_notfound_view(RemoveSlashNotFoundViewFactory())
config.add_static_view('static', 'static', cache_max_age=3600)
config.add_static_view('deform_static', 'deform:static')
config.add_static_view('files', settings['static_files'])
config.add_renderer('csv', '.tools.CSVRenderer')
for name, path, title in routes:
config.add_route(name, path)
config.scan()
app = config.make_wsgi_app()
from paste.translogger import TransLogger
app = TransLogger(app, setup_console_handler=False)
return app
import string
from decimal import Decimal
from datetime import date, datetime
def get_date_and_shop(session):
d = shared.get_answer(
'Data zakupów DD-MM-YYYY',
date.today().strftime('%d-%m-%Y'),
'^[0-4][0-9]-[0-1][0-9]-[0-9]{4}'
)
sklep = shared.choice('Wybierz sklep', session, m.Sklep, shared.enter_new(m.Sklep, session))
return datetime.strptime(d, '%d-%m-%Y'), sklep
if __name__ == '__main__':
m.init_model()
m.meta.create_all(m.meta.engine)
session = m.Session()
(data, sklep) = get_date_and_shop(session)
while True:
try:
kategoria = shared.choice('Wybierz kategorię: ', session, m.Kategoria, shared.enter_new(m.Kategoria))
podkategoria = shared.choice(
'Wybierz podkategorię: ',
session,
m.Podkategoria,
shared.enter_new(m.Podkategoria, kategoria),
query_processor = lambda x: x.filter(m.Podkategoria.kategoria_id == kategoria.id)
)
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu:
use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
train_dataset = data_manager.init_dataset(
root=args.root,
name='json',
phase='train'
)
valid_dataset = data_manager.init_dataset(
root=args.root,
name='json',
phase='valid'
)
test_dataset = data_manager.init_dataset(
root=args.root,
name='json',
phase='test'
)
test_mask = test_dataset.mask
pin_memory = True if use_gpu else False
trainloader = DataLoader(
JsonDataset(train_dataset),
num_workers=4,
batch_size=args.train_batch,
pin_memory=pin_memory,
drop_last=True
)
validloader = DataLoader(
JsonDataset(valid_dataset),
num_workers=4,
batch_size=args.test_batch,
pin_memory=pin_memory,
drop_last=True
)
testloader = DataLoader(
JsonDataset(test_dataset),
num_workers=4,
batch_size=args.test_batch,
pin_memory=pin_memory,
drop_last=True
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(
name=args.arch,
use_gpu=use_gpu)
print("Model size: {:.3f} M".format(count_num_param(model)))
model.init_weights()
criterion_label = LabelLoss()
optimizer = init_optim(
args.optim,
model.parameters(),
args.lr,
args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(
optimizer, milestones=args.stepsize, gamma=args.gamma)
start_epoch = args.start_epoch
if args.load_weights:
# load pretrained weights but ignore layers that don't match in size
print("Loading pretrained weights from '{}'".format(args.load_weights))
checkpoint = torch.load(args.load_weights)
pretrain_dict = checkpoint['state_dict']
model_dict = model.state_dict()
pretrain_dict = {k: v for k, v in pretrain_dict.items(
) if k in model_dict and model_dict[k].size() == v.size()}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
if args.resume:
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
print("Loaded checkpoint from '{}'".format(args.resume))
print("- start_epoch: {}".format(start_epoch))
# if use_gpu:
# str_ids = args.gpu_devices.split(',')
# gpu_ids = []
# for str_id in str_ids:
# id = int(str_id)
# if id >= 0:
# gpu_ids.append(id)
# model = nn.DataParallel(model, gpu_ids)
device = torch.device('cuda' if use_gpu and torch.cuda.is_available() else 'cpu')
model.to(device)
if args.evaluate:
print("Evaluate only")
start_evaluate_time = time.time()
test_thetas = evaluate(model, testloader, use_gpu, args.test_batch, test_mask)
evaluate_time = time.time() - start_evaluate_time
print('Evaluate: {} secs'.format(evaluate_time))
with open("auto_sample.csv", "r") as csvfiler:
with open("test_thetas5.csv", "w") as csvfilew:
reader = csv.reader(csvfiler)
for item in reader:
if reader.line_num == 1:
writer = csv.writer(csvfilew)
writer.writerow(['test_id', 'result'])
continue
writer = csv.writer(csvfilew)
writer.writerow([item[0], str(test_thetas[reader.line_num - 2])])
# writer.writerows(map(lambda x: [x], test_thetas))
return
start_time = time.time()
train_time = 0
best_label_loss = np.inf
best_epoch = 0
#
# print("==> Test")
# label_loss = test(model, validloader, criterion_label, use_gpu, args.test_batch)
# print("test label loss RMES() is {}".format(label_loss))
#
# print("==> Start training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(
epoch,
model,
criterion_label,
optimizer,
trainloader,
use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
# save model every epoch
if (epoch + 1) % args.save_step == 0:
print("==> Now save epoch {} \'s model".format(epoch + 1))
# if use_gpu:
# state_dict = model.state_dict() # module.
# else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'epoch': epoch
}, False, osp.join(args.save_dir, 'checkpoint_latest.pth'))
# test model every eval_step
if (epoch + 1) > args.start_eval and args.eval_step > 0 and (epoch + 1) % args.eval_step == 0 or \
(epoch + 1) == args.max_epoch:
print("==> Test")
label_loss = test(model, validloader, criterion_label, use_gpu, args.test_batch)
is_best = label_loss < best_label_loss
if is_best:
best_label_loss = label_loss
best_epoch = epoch + 1
# if use_gpu:
# state_dict = model.state_dict()
# else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'label_loss': label_loss,
'epoch': epoch,
}, is_best, osp.join(args.save_dir, 'checkpoint_ep' + str(epoch + 1) + '.pth')) # .pth.tar
print(
"==> Best Label Loss {:.3}, achieved at epoch {}".format(best_label_loss, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print(
"Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".format(
elapsed,
train_time))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_imgreid_dataset(
root=args.root, name=args.dataset, split_id=args.split_id,
cuhk03_labeled=args.cuhk03_labeled, cuhk03_classic_split=args.cuhk03_classic_split,
use_lmdb=args.use_lmdb,
)
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
trainloader = DataLoader(
ImageDataset(dataset.train, transform=transform_train,
use_lmdb=args.use_lmdb, lmdb_path=dataset.train_lmdb_path),
batch_size=args.train_batch, shuffle=True, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
queryloader = DataLoader(
ImageDataset(dataset.query, transform=transform_test,
use_lmdb=args.use_lmdb, lmdb_path=dataset.query_lmdb_path),
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
galleryloader = DataLoader(
ImageDataset(dataset.gallery, transform=transform_test,
use_lmdb=args.use_lmdb, lmdb_path=dataset.gallery_lmdb_path),
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch, num_classes=dataset.num_train_pids, loss={'xent'}, use_gpu=use_gpu)
print("Model size: {:.3f} M".format(count_num_param(model)))
criterion = CrossEntropyLabelSmooth(num_classes=dataset.num_train_pids, use_gpu=use_gpu)
optimizer = init_optim(args.optim, model.parameters(), args.lr, args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=args.stepsize, gamma=args.gamma)
start_epoch = args.start_epoch
if args.fixbase_epoch > 0:
optimizer_tmp = init_optim(args.optim, model.classifier.parameters(), args.fixbase_lr, args.weight_decay)
if args.resume:
print("Loading checkpoint from '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, use_gpu)
return
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
if args.fixbase_epoch > 0:
print("Train classifier for {} epochs while keeping base network frozen".format(args.fixbase_epoch))
for epoch in range(args.fixbase_epoch):
start_train_time = time.time()
train(epoch, model, criterion, optimizer_tmp, trainloader, use_gpu, freeze_bn=True)
train_time += round(time.time() - start_train_time)
del optimizer_tmp
print("Now open all layers for training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion, optimizer, trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
if (epoch+1) > args.start_eval and args.eval_step > 0 and (epoch+1) % args.eval_step == 0 or (epoch+1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best, osp.join(args.save_dir, 'checkpoint_ep' + str(epoch+1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print("Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".format(elapsed, train_time))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
print("Initializing dataset {}".format(args.dataset))
dataset = data_manager.init_imgreid_dataset(
root=args.root, name=args.dataset, split_id=args.split_id,
cuhk03_labeled=args.cuhk03_labeled, cuhk03_classic_split=args.cuhk03_classic_split,
use_lmdb=args.use_lmdb,
)
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
pin_memory = True if use_gpu else False
trainloader = DataLoader(
ImageDataset(dataset.train, transform=transform_train,
use_lmdb=args.use_lmdb, lmdb_path=dataset.train_lmdb_path),
batch_size=args.train_batch, shuffle=True, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
queryloader = DataLoader(
ImageDataset(dataset.query, transform=transform_test,
use_lmdb=args.use_lmdb, lmdb_path=dataset.query_lmdb_path),
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
galleryloader = DataLoader(
ImageDataset(dataset.gallery, transform=transform_test,
use_lmdb=args.use_lmdb, lmdb_path=dataset.gallery_lmdb_path),
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
print("Initializing model: {}".format(args.arch))
model = models.init_model(name=args.arch, num_classes=dataset.num_train_pids, loss={'xent'}, use_gpu=use_gpu)
print("Model size: {:.3f} M".format(count_num_param(model)))
criterion = CrossEntropyLabelSmooth(num_classes=dataset.num_train_pids, use_gpu=use_gpu)
optimizer = init_optim(args.optim, model.parameters(), args.lr, args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=args.stepsize, gamma=args.gamma)
start_epoch = args.start_epoch
if args.fixbase_epoch > 0:
if hasattr(model, 'classifier') and isinstance(model.classifier, nn.Module):
optimizer_tmp = init_optim(args.optim, model.classifier.parameters(), args.fixbase_lr, args.weight_decay)
else:
print("Warn: model has no attribute 'classifier' and fixbase_epoch is reset to 0")
args.fixbase_epoch = 0
if args.load_weights:
# load pretrained weights but ignore layers that don't match in size
print("Loading pretrained weights from '{}'".format(args.load_weights))
checkpoint = torch.load(args.load_weights)
pretrain_dict = checkpoint['state_dict']
model_dict = model.state_dict()
pretrain_dict = {k: v for k, v in pretrain_dict.items() if k in model_dict and model_dict[k].size() == v.size()}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
if args.resume:
checkpoint = torch.load(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
rank1 = checkpoint['rank1']
print("Loaded checkpoint from '{}'".format(args.resume))
print("- start_epoch: {}\n- rank1: {}".format(start_epoch, rank1))
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test(model, queryloader, galleryloader, use_gpu)
return
start_time = time.time()
train_time = 0
best_rank1 = -np.inf
best_epoch = 0
print("==> Start training")
if args.fixbase_epoch > 0:
print("Train classifier for {} epochs while keeping base network frozen".format(args.fixbase_epoch))
for epoch in range(args.fixbase_epoch):
start_train_time = time.time()
train(epoch, model, criterion, optimizer_tmp, trainloader, use_gpu, freeze_bn=True)
train_time += round(time.time() - start_train_time)
del optimizer_tmp
print("Now open all layers for training")
for epoch in range(start_epoch, args.max_epoch):
start_train_time = time.time()
train(epoch, model, criterion, optimizer, trainloader, use_gpu)
train_time += round(time.time() - start_train_time)
scheduler.step()
if (epoch+1) > args.start_eval and args.eval_step > 0 and (epoch+1) % args.eval_step == 0 or (epoch+1) == args.max_epoch:
print("==> Test")
rank1 = test(model, queryloader, galleryloader, use_gpu)
is_best = rank1 > best_rank1
if is_best:
best_rank1 = rank1
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best, osp.join(args.save_dir, 'checkpoint_ep' + str(epoch+1) + '.pth.tar'))
print("==> Best Rank-1 {:.1%}, achieved at epoch {}".format(best_rank1, best_epoch))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
train_time = str(datetime.timedelta(seconds=train_time))
print("Finished. Total elapsed time (h:m:s): {}. Training time (h:m:s): {}.".format(elapsed, train_time))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
if not args.evaluate:
sys.stdout = Logger(osp.join(args.save_dir, 'log_train.txt'))
else:
sys.stdout = Logger(osp.join(args.save_dir, 'log_test.txt'))
print("==========\nArgs:{}\n==========".format(args))
if use_gpu:
print("Currently using GPU {}".format(args.gpu_devices))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU (GPU is highly recommended)")
#print("Initializing dataset {}".format(args.dataset))
# dataset = data_manager.init_dataset(name=args.dataset)
transform_train = T.Compose([
T.Random2DTranslation(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_train_p = T.Compose([
T.Random2DTranslation(256, 128),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test = T.Compose([
T.Resize((args.height, args.width)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
transform_test_p = T.Compose([
T.Resize((256, 128)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
train_file = 'data/cuhk_train.pkl'
test_file = 'data/cuhk_test.pkl'
gallery_file = 'data/cuhk_gallery.pkl'
data_root = args.data_root
dataset_train = CUHKGroup(train_file, data_root, True, transform_train, transform_train_p)
dataset_test = CUHKGroup(test_file, data_root, False, transform_test, transform_test_p)
dataset_query = CUHKGroup(test_file, data_root, False, transform_test, transform_test_p)
dataset_gallery = CUHKGroup(gallery_file, data_root, False, transform_test, transform_test_p)
pin_memory = True if use_gpu else False
if args.xent_only:
trainloader = DataLoader(
dataset_train,
batch_size=args.train_batch, shuffle=True, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
else:
trainloader = DataLoader(
dataset_train,
sampler=RandomIdentitySampler(dataset_train, num_instances=args.num_instances),
batch_size=args.train_batch, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
queryloader = DataLoader(
dataset_test,
batch_size=args.test_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=False,
)
querygalleryloader = DataLoader(
dataset_query,
batch_size=args.gallery_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
galleryloader = DataLoader(
dataset_gallery,
batch_size=args.gallery_batch, shuffle=False, num_workers=args.workers,
pin_memory=pin_memory, drop_last=True,
)
print("Initializing model: {}".format(args.arch))
if args.xent_only:
# model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids, loss={'xent'})
model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids, loss={'xent'})
else:
# model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids, loss={'xent', 'htri'})
model = models.init_model(name=args.arch, num_classes=dataset_train.num_train_gids,
num_person_classes=dataset_train.num_train_pids, loss={'xent', 'htri'})
#criterion_xent = CrossEntropyLabelSmooth(num_classes=dataset_train.num_train_gids, use_gpu=use_gpu)
#criterion_xent_person = CrossEntropyLabelSmooth(num_classes=dataset_train.num_train_pids, use_gpu=use_gpu)
if os.path.exists(args.pretrained_model):
print("Loading checkpoint from '{}'".format(args.pretrained_model))
checkpoint = torch.load(args.pretrained_model)
model_dict = model.state_dict()
pretrain_dict = checkpoint['state_dict']
pretrain_dict = {k:v for k, v in pretrain_dict.items() if k in model_dict}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
criterion_xent = nn.CrossEntropyLoss(ignore_index=-1)
criterion_xent_person = nn.CrossEntropyLoss(ignore_index=-1)
criterion_htri = TripletLoss(margin=args.margin)
criterion_pair = ContrastiveLoss(margin=args.margin)
criterion_htri_filter = TripletLossFilter(margin=args.margin)
criterion_permutation = PermutationLoss()
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr,
weight_decay=args.weight_decay)
if args.stepsize > 0:
if args.warmup:
scheduler = WarmupMultiStepLR(optimizer, [200, 400, 600])
else:
scheduler = lr_scheduler.StepLR(optimizer, step_size=args.stepsize, gamma=args.gamma)
start_epoch = args.start_epoch
if use_gpu:
model = nn.DataParallel(model).cuda()
if args.evaluate:
print("Evaluate only")
test_gcn_person_batch(model, queryloader, querygalleryloader, galleryloader, args.pool, use_gpu)
#test_gcn_batch(model, queryloader, querygalleryloader, galleryloader, args.pool, use_gpu)
#test_gcn(model, queryloader, galleryloader, args.pool, use_gpu)
#test(model, queryloader, galleryloader, args.pool, use_gpu)
return
start_time = time.time()
best_rank1 = -np.inf
for epoch in range(start_epoch, args.max_epoch):
#print("==> Epoch {}/{} lr:{}".format(epoch + 1, args.max_epoch, scheduler.get_lr()[0]))
train_gcn(model, criterion_xent, criterion_xent_person, criterion_pair, criterion_htri_filter, criterion_permutation, optimizer, trainloader, use_gpu)
#train_gcn(model, criterion_xent, criterion_xent_person, criterion_pair, criterion_htri_filter, optimizer, trainloader, use_gpu)
#train(model, criterion_xent, criterion_htri, optimizer, trainloader, use_gpu)
if args.stepsize > 0: scheduler.step()
if args.eval_step > 0 and (epoch + 1) % args.eval_step == 0 or (epoch + 1) == args.max_epoch:
print("==> Test")
rank1 = test_gcn_person_batch(model, queryloader, querygalleryloader, galleryloader, args.pool, use_gpu)
#rank1 = test_gcn(model, queryloader, galleryloader, args.pool, use_gpu=False)
#rank1 = test(model, queryloader, galleryloader, args.pool, use_gpu)
is_best = rank1 > best_rank1
if is_best: best_rank1 = rank1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'rank1': rank1,
'epoch': epoch,
}, is_best, osp.join(args.save_dir, 'checkpoint_ep' + str(epoch + 1) + '.pth.tar'))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
