def main():
"""
Initializes and executes the program.
"""
login_sucessful = []
login_failed = []
login_skipped = []
version = check_revision(VERSION)
print("%s\n\n%s %s (%s)\n" %
(BANNER % tuple([color(_)
for _ in BANNER_PASSWORDS]), NAME, version, URL))
args = parse_args()
if args.update:
update()
exit()
sites = list_sites()
if args.list:
for _ in sites:
print("- %s" % _)
exit()
if not args.password and not args.load_file:
args.password = getpass("%s Please enter password:"******"(?P<type>[^:]+)://(?P<address>[^:]+)"
r":(?P<port>\d+)", args.proxy, re.I)
if match:
if match.group("type").upper() in ("HTTP", "HTTPS"):
proxy_host = "%s:%s" % (match.group("address"),
match.group("port"))
proxy_handler = ProxyHandler({
"http": proxy_host,
"https": proxy_host
})
else:
from thirdparty.socks import socks
if match.group("type").upper() == "SOCKS4":
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4,
match.group("address"),
int(match.group("port")), True)
elif match.group("type").upper() == "SOCKS5":
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5,
match.group("address"),
int(match.group("port")), True)
proxy_handler = None
else:
proxy_handler = ProxyHandler()
else:
proxy_handler = None
opener = build_opener(HTTPHandler(), HTTPSHandler(),
HTTPCookieProcessor(cookie_handler))
if proxy_handler:
opener.add_handler(proxy_handler)
install_opener(opener)
with open(USER_AGENTS_FILE, 'r') as ua_file:
args.user_agent = sample(ua_file.readlines(), 1)[0].strip()
if args.only:
sites = [site for site in sites if site in args.only]
elif args.exclude:
sites = [site for site in sites if site not in args.exclude]
print("%s Loaded %d %s to test." %
(INFO, len(sites), "site" if len(sites) == 1 else "sites"))
if args.load_file:
if not isfile(args.load_file):
print("%s could not find the file \"%s\"" %
(WARN, color(args.load_file)))
exit()
_ = sum(1 for line in open(args.load_file, "r"))
if _ < 1:
print("%s the file \"%s\" doesn't contain any valid credentials." %
(WARN, color(args.load_file)))
exit()
print("%s Loaded %d credential%s from \"%s\".\n" %
(INFO, _, "s" if _ != 1 else "", color(args.load_file)))
print("%s Starting tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW)))
if not exists(OUTPUT_DIR):
makedirs(OUTPUT_DIR)
log = Logger("%s/credmap" % OUTPUT_DIR)
log.open()
def get_targets():
"""
Retrieve and yield list of sites (targets) for testing.
"""
for site in sites:
_ = populate_site(site, args)
if not _:
continue
target = Website(_, {"verbose": args.verbose})
if not target.user_agent:
target.user_agent = args.user_agent
yield target
def login():
"""
Verify credentials for login and check if login was successful.
"""
if (target.username_or_email == "email" and not credentials["email"]
or target.username_or_email == "username"
and not credentials["username"]):
if args.verbose:
print(
"%s Skipping %s\"%s\" since "
"no \"%s\" was specified.\n" %
(INFO, "[%s:%s] on " %
(credentials["username"] or credentials["email"],
credentials["password"]) if args.load_file else "",
color(target.name), color(target.username_or_email, BW)))
login_skipped.append(target.name)
return
print("%s Testing %s\"%s\"..." %
(TEST, "[%s:%s] on " %
(credentials["username"] or credentials["email"],
credentials["password"]) if args.load_file else "",
color(target.name, BW)))
cookie_handler.clear()
if target.perform_login(credentials, cookie_handler):
log.write(">>> %s - %s:%s\n" %
(target.name, credentials["username"]
or credentials["email"], credentials["password"]))
login_sucessful.append(
"%s%s" % (target.name, " [%s:%s]" %
(credentials["username"] or credentials["email"],
credentials["password"]) if args.load_file else ""))
else:
login_failed.append(target.name)
if args.load_file:
if args.cred_format:
separators = [
re.escape(args.cred_format[1]),
re.escape(args.cred_format[3])
if len(args.cred_format) > 3 else "\n"
]
cred_format = re.match(r"(u|e|p)[^upe](u|e|p)(?:[^upe](u|e|p))?",
args.cred_format)
if not cred_format:
print("%s Could not parse --format: \"%s\"" %
(ERROR, color(args.cred_format, BW)))
exit()
cred_format = [
v.replace("e", "email").replace("u", "username").replace(
"p", "password") for v in cred_format.groups()
if v is not None
]
with open(args.load_file, "r") as load_list:
for user in load_list:
if args.cred_format:
match = re.match(
r"([^{0}]+){0}([^{1}]+)(?:{1}([^\n]+))?".format(
separators[0], separators[1]), user)
credentials = dict(zip(cred_format, match.groups()))
credentials["password"] = quote(credentials["password"])
if ("email" in credentials and not re.match(
r"^[A-Za-z0-9._%+-]+@(?:[A-Z"
r"a-z0-9-]+\.)+[A-Za-z]{2,12}$",
credentials["email"])):
print("%s Specified e-mail \"%s\" does not appear "
"to be correct. Skipping...\n" %
(WARN, color(credentials["email"], BW)))
continue
if "email" not in credentials:
credentials["email"] = None
elif "username" not in credentials:
credentials["username"] = None
else:
user = user.rstrip().split(":", 1)
if not user[0]:
if args.verbose:
print("%s Could not parse credentials: \"%s\"\n" %
(WARN, color(user, BW)))
continue
match = re.match(
r"^[A-Za-z0-9._%+-]+@(?:[A-Z"
r"a-z0-9-]+\.)+[A-Za-z]{2,12}$", user[0])
credentials = {
"email": user[0] if match else None,
"username": None if match else user[0],
"password": quote(user[1])
}
for target in get_targets():
login()
else:
credentials = {
"username": args.username,
"email": args.email,
"password": quote(args.password)
}
for target in get_targets():
login()
log.close()
if not args.verbose:
print()
if len(login_sucessful) > 0 or len(login_failed) > 0:
_ = "%s/%s" % (color(len(login_sucessful), BW),
color(len(login_sucessful) + len(login_failed), BW))
sign = PLUS if len(login_sucessful) > (len(login_failed) +
len(login_skipped)) else INFO
print(
"%s Succesfully logged in%s." %
(sign, " with %s credentials on the list." %
_ if args.load_file else "to %s websites." % _), )
print("%s An overall success rate of %s.\n" %
(sign,
color(
"%%%s" % (100 * len(login_sucessful) /
(len(login_sucessful) + len(login_failed))), BW)))
if len(login_sucessful) > 0:
print("%s The provided credentials worked on the following website%s: "
"%s\n" % (PLUS, "s" if len(login_sucessful) != 1 else "",
", ".join(login_sucessful)))
print("%s Finished tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW)))
def run(opt):
torch.manual_seed(opt.seed)
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
print('Setting up data...')
Dataset = get_dataset(opt.dataset, opt.task,
opt.multi_scale) # if opt.task==mot -> JointDataset
f = open(opt.data_cfg
) # choose which dataset to train '../src/lib/cfg/mot15.json',
data_config = json.load(f)
trainset_paths = data_config['train'] # 训练集路径
dataset_root = data_config['root'] # 数据集所在目录
print("Dataset root: %s" % dataset_root)
f.close()
# Image data transformations
transforms = T.Compose([T.ToTensor()])
# Dataset
dataset = Dataset(opt=opt,
root=dataset_root,
paths=trainset_paths,
img_size=opt.input_wh,
augment=True,
transforms=transforms)
opt = opts().update_dataset_info_and_set_heads(opt, dataset)
print("opt:\n", opt)
logger = Logger(opt)
# os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str # 多GPU训练
# print("opt.gpus_str: ", opt.gpus_str)
opt.device = torch.device('cuda:0' if opt.gpus[0] >= 0 else 'cpu') # 设置GPU
#opt.device = device #NC UPDATE - fallback to original fairmot
#opt.gpus = my_visible_devs #NC UPDATE
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
# 初始化优化器
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
# Get dataloader
if opt.is_debug:
if opt.multi_scale:
train_loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=opt.batch_size,
shuffle=False,
pin_memory=True,
drop_last=True) # debug时不设置线程数(即默认为0)
else:
train_loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=opt.batch_size,
shuffle=True,
pin_memory=True,
drop_last=True) # debug时不设置线程数(即默认为0)
else:
if opt.multi_scale:
train_loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
else:
train_loader = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=opt.batch_size,
shuffle=True,
pin_memory=True,
drop_last=True) # debug时不设置线程数(即默认为0)
print('Starting training...')
Trainer = train_factory[opt.task]
trainer = Trainer(opt=opt, model=model, optimizer=optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
# trainer.set_device(opt.gpus, opt.chunk_sizes, device) #NC UPDATE
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
# Train an epoch
log_dict_train, _ = trainer.train(epoch, train_loader)
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
else: # mcmot_last_track or mcmot_last_det
if opt.id_weight > 0: # do tracking(detection and re-id)
save_model(
os.path.join(opt.save_dir,
'mcmot_last_track_' + opt.arch + '.pth'),
epoch, model, optimizer)
else: # only do detection
# save_model(os.path.join(opt.save_dir, 'mcmot_last_det_' + opt.arch + '.pth'),
# epoch, model, optimizer)
save_model(
os.path.join(opt.save_dir,
'mcmot_last_det_' + opt.arch + '.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if epoch % 10 == 0:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
logger.close()
def main(opt):
torch.manual_seed(opt.seed)
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
Dataset = get_dataset(opt.dataset, opt.task)
opt = opts().update_dataset_info_and_set_heads(opt, Dataset)
print(opt)
logger = Logger(opt)
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
Trainer = train_factory[opt.task]
trainer = Trainer(opt, model, optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
print('Setting up data...')
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=0,
pin_memory=True)
if opt.test:
_, preds = trainer.val(0, val_loader)
val_loader.dataset.run_eval(preds, opt.save_dir)
return
train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'),
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
print('Starting training...')
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
log_dict_train, _ = trainer.train(epoch, train_loader)
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
with torch.no_grad():
log_dict_val, preds = trainer.val(epoch, val_loader)
for k, v in log_dict_val.items():
logger.scalar_summary('val_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if log_dict_val[opt.metric] < best:
best = log_dict_val[opt.metric]
save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch,
model)
else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.close()
def main(opt, opt_t):
torch.manual_seed(opt.seed)
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
if opt.target_dataset:
Dataset_target = get_dataset(opt_t.target_dataset, opt_t.task)
opt_t = opts().update_dataset_info_and_set_heads(
opt_t, Dataset_target) # target dataset
Dataset_source = get_dataset(opt.source_dataset, opt.task)
opt = opts().update_dataset_info_and_set_heads(
opt, Dataset_source) # source dataset
print(opt)
logger = Logger(opt) # record
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv) # create model
optimizer = torch.optim.Adam(model.parameters(),
opt.lr) # create optimizer
start_epoch = 0
if opt.load_model != '': # load model
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
Trainer = train_factory[opt.task] # set trainer function
trainer = Trainer(opt, model, optimizer) # initial trainer
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
print('Setting up source_val data...')
val_source_loader = torch.utils.data.DataLoader(Dataset_source(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=0,
pin_memory=True)
if opt.test:
_, preds = trainer.val(0, val_source_loader)
val_source_loader.dataset.run_eval(preds, opt.save_dir)
return
# source loader
print('Setting up source_train data...')
train_source_loader = torch.utils.data.DataLoader(
Dataset_source(opt, 'train'), # modify SOURCE dataset parameters
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
if opt.target_dataset:
# target loader
print('Setting up target_train data...')
train_target_loader = torch.utils.data.DataLoader(
Dataset_target(opt_t, 'train'), # modify TARGET dataset parameters
batch_size=opt_t.batch_size,
shuffle=True,
num_workers=opt_t.num_workers,
pin_memory=True,
drop_last=True)
print('DA MODE')
else:
train_target_loader = None
print('Starting training...')
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
log_dict_train, _ = trainer.train(epoch, train_source_loader,
train_target_loader) # do train
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items(): # log information
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
# if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
# save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), # save last-model
# epoch, model, optimizer)
# with torch.no_grad():
# log_dict_val, preds = trainer.val(epoch, val_source_loader) # cal val-set loss
# for k, v in log_dict_val.items():
# logger.scalar_summary('val_{}'.format(k), v, epoch)
# logger.write('{} {:8f} | '.format(k, v))
# if log_dict_val[opt.metric] < best:
# best = log_dict_val[opt.metric]
# save_model(os.path.join(opt.save_dir, 'model_best.pth'), # save best-model
# epoch, model)
# else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step: # update learning rate
save_model(
os.path.join(
opt.save_dir,
'model_{}.pth'.format(epoch)), # save lr_step-model
epoch,
model,
optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.close()
def main(opt):
if opt.disable_cudnn:
torch.backends.cudnn.enabled = False
print('Cudnn is disabled.')
logger = Logger(opt)
opt.device = torch.device('cuda:{}'.format(opt.gpus[0]))
Dataset = dataset_factory[opt.dataset]
train, val = task_factory[opt.task]
model, optimizer, start_epoch = create_model(opt)
if len(opt.gpus) > 1:
model = torch.nn.DataParallel(model,
device_ids=opt.gpus).cuda(opt.device)
else:
model = model.cuda(opt.device)
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
if opt.test:
log_dict_train, preds = val(0, opt, val_loader, model)
sio.savemat(os.path.join(opt.save_dir, 'preds.mat'),
mdict={'preds': preds})
return
train_loader = torch.utils.data.DataLoader(
Dataset(opt, 'train'),
batch_size=opt.batch_size * len(opt.gpus),
shuffle=True, # if opt.debug == 0 else False,
num_workers=opt.num_workers,
pin_memory=True)
best = -1
for epoch in range(start_epoch, opt.num_epochs + 1):
mark = epoch if opt.save_all_models else 'last'
log_dict_train, _ = train(epoch, opt, train_loader, model, optimizer)
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
log_dict_val, preds = val(epoch, opt, val_loader, model)
for k, v in log_dict_val.items():
logger.scalar_summary('val_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if log_dict_val[opt.metric] > best:
best = log_dict_val[opt.metric]
save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch,
model)
else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.close()
def main(opt):
torch.manual_seed(opt.seed)
# benchmark=True 自动寻找最适合当前配置的高效算法，来达到优化运行效率的wento
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
Dataset = get_dataset(opt.dataset, opt.task)
opt = opts().update_dataset_info_and_set_heads(opt, Dataset)
print(opt)
logger = Logger(opt)
def adjust_learning_rate(optimizer, epoch):
# use warmup
if epoch < 5:
lr = opt.lr * ((epoch + 1) / 5)
else:
# use cosine lr
PI = 3.14159
lr = opt.lr * 0.5 * (1 + math.cos(epoch * PI / 20))
# print(1111)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
# optimizer = torch.optim.Adam(model.parameters(), opt.lr)
optimizer = torch.optim.SGD(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
Trainer = train_factory[opt.task]
trainer = Trainer(opt, model, optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
print('Setting up data...')
# num_worker=0是主进程读取; >0使用多进程读取，子进程读取数据时，训练程序会卡住，GPU utils为0，
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
if opt.test:
_, preds = trainer.val(0, val_loader)
val_loader.dataset.run_eval(preds, opt.save_dir)
return
train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'),
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
print('Starting training...')
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
adjust_learning_rate(optimizer, epoch)
mark = epoch if opt.save_all else 'last'
log_dict_train, _ = trainer.train(epoch, train_loader)
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % 2 == 0 and epoch > 10:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
with torch.no_grad():
log_dict_val, preds = trainer.val(epoch, val_loader)
for k, v in log_dict_val.items():
logger.scalar_summary('val_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if log_dict_val[opt.metric] < best:
best = log_dict_val[opt.metric]
save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch,
model)
# test(opt)
# opt.model = None
if epoch > 20:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
# elif 80 < epoch <=100 and epoch % 3 == 0:
# save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
# epoch, model, optimizer)
logger.write('\n')
# if epoch in opt.lr_step:
# save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
# epoch, model, optimizer)
# lr = opt.lr * (0.1 ** (opt.lr_step.index(epoch) + 1))
# print('Drop LR to', lr)
# for param_group in optimizer.param_groups:
# param_group['lr'] = lr
print('Epoch is Finished')
logger.close()
def main():
"""
Initializes and executes the program.
"""
login_sucessful = []
login_failed = []
login_skipped = []
version = check_revision(VERSION)
print("%s\n\n%s %s (%s)\n" % (
BANNER % tuple([color(_) for _ in BANNER_PASSWORDS]),
NAME, version, URL))
args = parse_args()
if args.update:
update()
exit()
sites = list_sites()
if args.list:
for _ in sites:
print("- %s" % _)
exit()
if not args.password and not args.load_file:
args.password = getpass("%s Please enter password:"******"(?P<type>[^:]+)://(?P<address>[^:]+)"
r":(?P<port>\d+)", args.proxy, re.I)
if match:
if match.group("type").upper() in ("HTTP", "HTTPS"):
proxy_host = "%s:%s" % (match.group("address"),
match.group("port"))
proxy_handler = ProxyHandler({"http": proxy_host,
"https": proxy_host})
else:
from thirdparty.socks import socks
if match.group("type").upper() == "SOCKS4":
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4,
match.group("address"),
int(match.group("port")), True)
elif match.group("type").upper() == "SOCKS5":
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5,
match.group("address"),
int(match.group("port")), True)
proxy_handler = None
else:
proxy_handler = ProxyHandler()
else:
proxy_handler = None
opener = build_opener(HTTPHandler(), HTTPSHandler(),
HTTPCookieProcessor(cookie_handler))
if proxy_handler:
opener.add_handler(proxy_handler)
install_opener(opener)
with open(USER_AGENTS_FILE, 'r') as ua_file:
args.user_agent = sample(ua_file.readlines(), 1)[0].strip()
if args.only:
sites = [site for site in sites if site in args.only]
elif args.exclude:
sites = [site for site in sites if site not in args.exclude]
print("%s Loaded %d %s to test." %
(INFO, len(sites), "site" if len(sites) == 1 else "sites"))
if args.load_file:
if not isfile(args.load_file):
print("%s could not find the file \"%s\"" %
(WARN, color(args.load_file)))
exit()
_ = sum(1 for line in open(args.load_file, "r"))
if _ < 1:
print("%s the file \"%s\" doesn't contain any valid credentials." %
(WARN, color(args.load_file)))
exit()
print("%s Loaded %d credential%s from \"%s\".\n" %
(INFO, _, "s" if _ != 1 else "", color(args.load_file)))
print("%s Starting tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW)))
if not exists(OUTPUT_DIR):
makedirs(OUTPUT_DIR)
log = Logger("%s/credmap" % OUTPUT_DIR)
log.open()
def get_targets():
"""
Retrieve and yield list of sites (targets) for testing.
"""
for site in sites:
_ = populate_site(site, args)
if not _:
continue
target = Website(_, {"verbose": args.verbose})
if not target.user_agent:
target.user_agent = args.user_agent
yield target
def login():
"""
Verify credentials for login and check if login was successful.
"""
if(target.username_or_email == "email" and not
credentials["email"] or
target.username_or_email == "username" and not
credentials["username"]):
if args.verbose:
print("%s Skipping %s\"%s\" since "
"no \"%s\" was specified.\n" %
(INFO, "[%s:%s] on " %
(credentials["username"] or
credentials["email"], credentials["password"]) if
args.load_file else "", color(target.name),
color(target.username_or_email, BW)))
login_skipped.append(target.name)
return
print("%s Testing %s\"%s\"..." %
(TEST, "[%s:%s] on " % (credentials["username"] or
credentials["email"],
credentials["password"]) if
args.load_file else "", color(target.name, BW)))
cookie_handler.clear()
if target.perform_login(credentials, cookie_handler):
log.write(">>> %s - %s:%s\n" %
(target.name, credentials["username"] or
credentials["email"], credentials["password"]))
login_sucessful.append("%s%s" %
(target.name, " [%s:%s]" %
(credentials["username"] or
credentials["email"],
credentials["password"]) if
args.load_file else ""))
else:
login_failed.append(target.name)
if args.load_file:
if args.cred_format:
separators = [re.escape(args.cred_format[1]),
re.escape(args.cred_format[3]) if
len(args.cred_format) > 3 else "\n"]
cred_format = re.match(r"(u|e|p)[^upe](u|e|p)(?:[^upe](u|e|p))?",
args.cred_format)
if not cred_format:
print("%s Could not parse --format: \"%s\""
% (ERROR, color(args.cred_format, BW)))
exit()
cred_format = [v.replace("e", "email")
.replace("u", "username")
.replace("p", "password")
for v in cred_format.groups() if v is not None]
with open(args.load_file, "r") as load_list:
for user in load_list:
if args.cred_format:
match = re.match(r"([^{0}]+){0}([^{1}]+)(?:{1}([^\n]+))?"
.format(separators[0], separators[1]),
user)
credentials = dict(zip(cred_format, match.groups()))
credentials["password"] = quote(
credentials["password"])
if("email" in credentials and
not re.match(r"^[A-Za-z0-9._%+-]+@(?:[A-Z"
r"a-z0-9-]+\.)+[A-Za-z]{2,12}$",
credentials["email"])):
print("%s Specified e-mail \"%s\" does not appear "
"to be correct. Skipping...\n" % (WARN, color(
credentials["email"], BW)))
continue
if "email" not in credentials:
credentials["email"] = None
elif "username" not in credentials:
credentials["username"] = None
else:
user = user.rstrip().split(":", 1)
if not user[0]:
if args.verbose:
print("%s Could not parse credentials: \"%s\"\n" %
(WARN, color(user, BW)))
continue
match = re.match(r"^[A-Za-z0-9._%+-]+@(?:[A-Z"
r"a-z0-9-]+\.)+[A-Za-z]{2,12}$", user[0])
credentials = {"email": user[0] if match else None,
"username": None if match else user[0],
"password": quote(user[1])}
for target in get_targets():
login()
else:
credentials = {"username": args.username, "email": args.email,
"password": quote(args.password)}
for target in get_targets():
login()
log.close()
if not args.verbose:
print()
if len(login_sucessful) > 0 or len(login_failed) > 0:
_ = "%s/%s" % (color(len(login_sucessful), BW),
color(len(login_sucessful) + len(login_failed), BW))
sign = PLUS if len(login_sucessful) > (len(login_failed) +
len(login_skipped)) else INFO
print("%s Succesfully logged in%s." %
(sign, " with %s credentials on the list." % _ if args.load_file
else "to %s websites." % _),)
print("%s An overall success rate of %s.\n" %
(sign, color("%%%s" % (100 * len(login_sucessful) /
(len(login_sucessful) +
len(login_failed))), BW)))
if len(login_sucessful) > 0:
print("%s The provided credentials worked on the following website%s: "
"%s\n" % (PLUS, "s" if len(login_sucessful) != 1 else "",
", ".join(login_sucessful)))
print("%s Finished tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW)))
def main(opt):
torch.manual_seed(
opt.seed
) # opt.seed: default=317 ；加上torch.manual_seed这个函数调用的话，打印出来的随机数每次都一样。
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
Dataset = get_dataset(
opt.dataset, opt.task
) # opt.dataset = coco, opt.task = ctdet (| ddd | multi_pose | exdet)
opt = opts().update_dataset_info_and_set_heads(opt, Dataset)
print(opt)
logger = Logger(opt)
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
Trainer = train_factory[opt.task]
trainer = Trainer(opt, model, optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
print('Setting up data...')
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True) # modified by zy
# val_loader = torch.utils.data.DataLoader(Dataset(opt, 'test'), batch_size=1, shuffle=False, num_workers=1,pin_memory=True)
if opt.test:
_, preds = trainer.val(0, val_loader)
val_loader.dataset.run_eval(preds, opt.save_dir)
return
train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'),
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
output_choice_log = '/home/zy/zy/2new_network/CenterNet-master/output_choice.log'
if os.path.exists(output_choice_log):
os.remove(output_choice_log)
print('Starting training...')
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
try:
log_dict_train, _ = trainer.train(
epoch, train_loader
) # !!!!!!!! train = self.run_epoch('train', epoch, data_loader)
except Exception as e: # 如果发生异常，那就返回预设的loss值
print('Error_train!!!', e)
print(traceback.format_exc())
continue
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
with torch.no_grad():
try:
log_dict_val, preds = trainer.val(epoch, val_loader)
except Exception as e: # 如果发生异常，那就返回预设的loss值
print('Error_train!!!', e)
print(traceback.format_exc())
continue
for k, v in log_dict_val.items():
logger.scalar_summary('val_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if log_dict_val[opt.metric] < best:
best = log_dict_val[opt.metric]
save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch,
model)
else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.close()
def main(opt):
torch.manual_seed(opt.seed)
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
print('Setting up data...')
Dataset = get_dataset(opt.task)
f = open(opt.data_cfg)
data_config = json.load(f)
trainset_paths = data_config['train']
dataset_root = data_config['root']
f.close()
transforms = T.Compose([T.ToTensor()])
dataset = Dataset(opt,
dataset_root,
trainset_paths, (1088, 608),
augment=True,
transforms=transforms)
opt = opts().update_dataset_info_and_set_heads(opt, dataset)
print(opt)
logger = Logger(opt)
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
# Get dataloader
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
print('Starting training...')
Trainer = train_factory[opt.task]
trainer = Trainer(opt, model, optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
log_dict_train, _ = trainer.train(epoch, train_loader)
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if epoch % 5 == 0:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
logger.close()
