def CommitMatch(self,db, doValidation=True):
self.ParseSpringOutput()
ladder = db.GetLadder(self.ladder_id )
gameid = self.gameid
if doValidation and not self.CheckValidSetup( db ):
raise InvalidOptionSetup( gameid, self.ladder_id )
session = db.session()
match = Match()
match.date = datetime.datetime.now()
match.modname = ''
replayzip = os.path.basename(self.replay).replace(".sdf",".zip")
base = tasbot.config.Config().get('ladder','base_dir')
replayzip = os.path.join( base, 'demos', replayzip)
mkdir_p(replayzip)
with zipfile.ZipFile(replayzip, mode='w') as f_out:
f_out.write(self.replay, compress_type=zipfile.ZIP_DEFLATED)
match.replay = replayzip
match.game_id = gameid
match.ladder_id = ladder.id
match.last_frame = self.game_over
match.duration = datetime.timedelta( seconds=float(match.last_frame) / 30.0 )
session.add( match )
session.commit()
#session.refresh()
matchid = match.id
match.mapname = ''
for key,val in self.options.iteritems():
s = MatchSetting()
s.key = key
s.value = val
if key == "mapname":
query = session.query( Map ).filter( Map.name == val )
if query.count() == 0:
import xmlrpcdl
db_map = xmlrpcdl.DownloadQueue().add_map(val)
session.add(db_map)
session.commit()
else:
db_map = query.one()
match.mapname = db_map.name
session.add( match )
session.commit()
if key == "modname":
match.modname = val
session.add( match )
session.commit()
s.match_id = match.id
session.add( s )
#session.commit()
for key,val in self.restrictions.iteritems():
s = MatchSetting()
s.key = key
s.value = val
s.match_id = match.id
session.add( s )
session.commit()
self.CommitPlayerResults(session,match)
session.close()
GlobalRankingAlgoSelector.GetInstance( ladder.ranking_algo_id ).Update( ladder.id, match, db )
return matchid
def add_map(self,mapname):
pattern = self._searchstring(mapname)
result = self._proxy.springfiles.search(pattern)[0]
name = result['name']
nmap = Map()
meta = result['metadata']
nmap.startpos = [(f['x'],f['z']) for f in meta['StartPos'] ]
nmap.name = name
nmap.md5 = result['md5']
try:
imgurl = result['mapimages'][0]
basedir = nmap.basedir(globe.config)
basename = nmap.name + imgurl[-4:]
local_fn = os.path.join( basedir, '1024', basename )
globe.mkdir_p(local_fn)
urllib.urlretrieve (imgurl, local_fn)
img = Image.open(local_fn)
for i in [128,256,512]:
resized_img = img.resize((i, i), Image.ANTIALIAS)
fn = os.path.join(basedir, str(i), basename)
myutils.mkdir_p(fn)
resized_img.save(fn)
nmap.minimap = basename
except Exception, e:
globe.Log.exception(e)
globe.Log.error('download for map %s failed'%nmap.name)
def save_state(self, conf, to_save_folder=False, model_only=False):
if to_save_folder:
save_path = conf.save_path
else:
save_path = conf.model_path
myutils.mkdir_p(save_path, delete=False)
torch.save(
self.model.state_dict(),
save_path /
('model_{}_step:{}.pth'.format(get_time(), self.step)))
if not model_only:
torch.save(
self.optimizer.state_dict(),
save_path /
('optimizer_{}_step:{}.pth'.format(get_time(), self.step)))
def get_config():
parser = argparse.ArgumentParser('argument for training')
parser.add_argument('--use_dataset',
type=str,
default='Cars',
choices=['CUB', 'Cars', 'SOP', 'Inshop'])
# batch
parser.add_argument('--batch_size', type=int, default=120)
parser.add_argument('--instances', type=int, default=3)
# optimization
parser.add_argument('--lr', type=float, default=0.0)
parser.add_argument('--lr_p', type=float, default=0.0)
parser.add_argument('--lr_gamma', type=float, default=0.0)
# model dataset
parser.add_argument('--freeze_bn', type=int, default=1)
# method
parser.add_argument('--use_loss',
type=str,
default='triplet',
choices=['triplet', 'n-npair', 'semihtriplet', 'ms'])
parser.add_argument('--sec_wei', type=float, default=0.0)
parser.add_argument('--norm_momentum', type=float, default=1.0)
parser.add_argument('--l2reg_wei', type=float, default=0.0)
parser.add_argument('--test_sop_model', type=str, default='')
conf = parser.parse_args()
conf.num_devs = 1
if conf.use_dataset == 'CUB':
conf.lr = 1.0e-5 if conf.lr == 0 else conf.lr
conf.lr_p = 0.5e-5 if conf.lr_p == 0 else conf.lr_p
conf.weight_decay = 0.5 * 5e-3
conf.start_step = 0
conf.lr_gamma = 0.1 if conf.lr_gamma == 0 else conf.lr_gamma
if conf.use_loss == 'ms':
conf.step_milestones = [3000, 6000, 9000]
else:
conf.step_milestones = [5000, 9000, 9000]
conf.steps = 8000
elif conf.use_dataset == 'Cars':
conf.lr = 1e-5 if conf.lr == 0 else conf.lr
conf.lr_p = 1e-5 if conf.lr_p == 0 else conf.lr_p
conf.weight_decay = 0.5 * 5e-3
conf.start_step = 0
if conf.lr_gamma == 0.1:
conf.step_milestones = [2000, 9000, 9000]
elif conf.lr_gamma == 0.5:
conf.step_milestones = [4000, 6000, 9000]
conf.steps = 8000
elif conf.use_dataset == 'SOP':
conf.lr = 2.5e-4 if conf.lr == 0 else conf.lr
conf.lr_p = 0.5e-4 if conf.lr_p == 0 else conf.lr_p
conf.weight_decay = 1e-5
conf.start_step = 0
conf.lr_gamma = 0.1 if conf.lr_gamma == 0 else conf.lr_gamma
conf.step_milestones = [6e3, 18e3, 35e3]
conf.steps = 12e3
elif conf.use_dataset == 'Inshop':
conf.lr = 5e-4 if conf.lr == 0 else conf.lr
conf.lr_p = 1e-4 if conf.lr_p == 0 else conf.lr_p
conf.weight_decay = 1e-5
conf.start_step = 0
conf.lr_gamma = 0.1 if conf.lr_gamma == 0 else conf.lr_gamma
conf.step_milestones = [6e3, 18e3, 35e3]
conf.steps = 12e3
conf.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
now_time = datetime.datetime.now().strftime('%m%d_%H%M')
conf_work_path = 'work_space/' + conf.use_dataset + '_' + now_time
myutils.mkdir_p(conf_work_path, delete=True)
myutils.set_file_logger(work_dir=conf_work_path, log_level=logging.DEBUG)
sys.stdout = myutils.Logger(conf_work_path + '/log-prt')
sys.stderr = myutils.Logger(conf_work_path + '/log-prt-err')
path0, path1 = conf_work_path.split('/')
conf.log_path = Path(path0) / 'logs' / path1 / 'log'
conf.work_path = Path(conf_work_path)
conf.model_path = conf.work_path / 'models'
conf.save_path = conf.work_path / 'save'
conf.start_eval = False
conf.num_workers = 8
conf.bninception_pretrained_model_path = './pretrained_models/bn_inception-52deb4733.pth'
conf.transform_dict = {}
conf.use_simple_aug = False
conf.transform_dict['rand-crop'] = \
transforms.Compose([
transforms.Resize(size=(256, 256)) if conf.use_simple_aug else transforms.Resize(size=256),
transforms.RandomCrop((227, 227)) if conf.use_simple_aug else transforms.RandomResizedCrop(
scale=[0.16, 1],
size=227
),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[123 / 255.0, 117 / 255.0, 104 / 255.0],
std=[1.0 / 255, 1.0 / 255, 1.0 / 255]),
transforms.Lambda(lambda x: x[[2, 1, 0], ...]) #to BGR
])
conf.transform_dict['center-crop'] = \
transforms.Compose([
transforms.Resize(size=(256, 256)) if conf.use_simple_aug else transforms.Resize(size=256),
transforms.CenterCrop(227),
transforms.ToTensor(),
transforms.Normalize(mean=[123 / 255.0, 117 / 255.0, 104 / 255.0],
std=[1.0 / 255, 1.0 / 255, 1.0 / 255]),
transforms.Lambda(lambda x: x[[2, 1, 0], ...]) #to BGR
])
return conf
from beaker.util import parse_cache_config_options
import os
from bottle import static_file
from tasbot.customlog import Log
from tasbot.config import Config
from disqus import Disqus
import ladderdb
from myutils import mkdir_p
cache_opts = {
'cache.type': 'memory',
'cache.data_dir': 'tmp/cache/data',
'cache.lock_dir': 'tmp/cache/lock'
}
config = Config( 'Main.conf' )
Log.init( 'website.log' )
db = ladderdb.LadderDB(config.get('tasbot','alchemy-uri'))
env = Environment(loader=FileSystemLoader('templates'))
staging = config.get_bool('tasbot','staging')
cache = CacheManager(**parse_cache_config_options(cache_opts))
disqus = Disqus(config,cache)
mkdir_p(config.get('ladder','base_dir'))
def local_file(filename, sub, **kwargs):
path = os.path.join(config.get('ladder','base_dir'),sub)
return static_file(filename, root=path)
def __init__(self, conf, inference=False):
logging.info(f'metric learner use {conf}')
self.model = torch.nn.DataParallel(BNInception()).cuda()
logging.info(f'model generated')
if not inference:
if conf.use_dataset == 'CUB':
self.dataset = MSBaseDataSet(conf, './datasets/CUB_200_2011/cub_train.txt',
transform=conf.transform_dict['rand-crop'], mode='RGB')
elif conf.use_dataset == 'Cars':
self.dataset = MSBaseDataSet(conf, './datasets/CARS196/cars_train.txt',
transform=conf.transform_dict['rand-crop'], mode='RGB')
elif conf.use_dataset == 'SOP':
self.dataset = MSBaseDataSet(conf, './datasets/SOP/sop_train.txt',
transform=conf.transform_dict['rand-crop'], mode='RGB')
elif conf.use_dataset == 'Inshop':
self.dataset = MSBaseDataSet(conf, './datasets/Inshop/inshop_train.txt',
transform=conf.transform_dict['rand-crop'], mode='RGB')
self.loader = DataLoader(
self.dataset, batch_size=conf.batch_size, num_workers=conf.num_workers,
shuffle=False, sampler=RandomIdSampler(conf, self.dataset.label_index_dict), drop_last=True,
pin_memory=True,
)
self.class_num = self.dataset.num_cls
self.img_num = self.dataset.num_train
myutils.mkdir_p(conf.log_path, delete=True)
self.writer = SummaryWriter(str(conf.log_path))
self.step = 0
self.head_npair = NpairLoss().to(conf.device)
self.head_semih_triplet = TripletSemihardLoss().to(conf.device)
self.head_triplet = TripletLoss(instance=conf.instances).to(conf.device)
self.head_multisimiloss = MultiSimilarityLoss().to(conf.device)
logging.info('model heads generated')
backbone_bn_para, backbone_wo_bn_para = [
[p for k, p in self.model.named_parameters() if
('bn' in k) == is_bn and ('head' in k) == False] for is_bn in [True, False]]
head_bn_para, head_wo_bn_para = [
[p for k, p in self.model.module.head.named_parameters() if
('bn' in k) == is_bn] for is_bn in [True, False]]
self.optimizer = optim.Adam([
{'params': backbone_bn_para if conf.freeze_bn==False else [], 'lr': conf.lr_p},
{'params': backbone_wo_bn_para, 'weight_decay': conf.weight_decay, 'lr': conf.lr_p},
{'params': head_bn_para, 'lr': conf.lr},
{'params': head_wo_bn_para, 'weight_decay': conf.weight_decay, 'lr': conf.lr},
])
logging.info(f'{self.optimizer}, optimizers generated')
if conf.use_dataset=='CUB' or conf.use_dataset=='Cars':
self.board_loss_every = 20
self.evaluate_every = 100
self.save_every = 1000
elif conf.use_dataset=='Inshop':
self.board_loss_every = 20
self.evaluate_every = 200
self.save_every = 2000
else:
self.board_loss_every = 20
self.evaluate_every = 500
self.save_every = 2000
