def main(): """ Initializes and executes the program. """ login_sucessful = [] login_failed = [] login_skipped = [] version = check_revision(VERSION) print("%s\n\n%s %s (%s)\n" % (BANNER % tuple([color(_) for _ in BANNER_PASSWORDS]), NAME, version, URL)) args = parse_args() if args.update: update() exit() sites = list_sites() if args.list: for _ in sites: print("- %s" % _) exit() if not args.password and not args.load_file: args.password = getpass("%s Please enter password:"******"(?P<type>[^:]+)://(?P<address>[^:]+)" r":(?P<port>\d+)", args.proxy, re.I) if match: if match.group("type").upper() in ("HTTP", "HTTPS"): proxy_host = "%s:%s" % (match.group("address"), match.group("port")) proxy_handler = ProxyHandler({ "http": proxy_host, "https": proxy_host }) else: from thirdparty.socks import socks if match.group("type").upper() == "SOCKS4": socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4, match.group("address"), int(match.group("port")), True) elif match.group("type").upper() == "SOCKS5": socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, match.group("address"), int(match.group("port")), True) proxy_handler = None else: proxy_handler = ProxyHandler() else: proxy_handler = None opener = build_opener(HTTPHandler(), HTTPSHandler(), HTTPCookieProcessor(cookie_handler)) if proxy_handler: opener.add_handler(proxy_handler) install_opener(opener) with open(USER_AGENTS_FILE, 'r') as ua_file: args.user_agent = sample(ua_file.readlines(), 1)[0].strip() if args.only: sites = [site for site in sites if site in args.only] elif args.exclude: sites = [site for site in sites if site not in args.exclude] print("%s Loaded %d %s to test." % (INFO, len(sites), "site" if len(sites) == 1 else "sites")) if args.load_file: if not isfile(args.load_file): print("%s could not find the file \"%s\"" % (WARN, color(args.load_file))) exit() _ = sum(1 for line in open(args.load_file, "r")) if _ < 1: print("%s the file \"%s\" doesn't contain any valid credentials." % (WARN, color(args.load_file))) exit() print("%s Loaded %d credential%s from \"%s\".\n" % (INFO, _, "s" if _ != 1 else "", color(args.load_file))) print("%s Starting tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW))) if not exists(OUTPUT_DIR): makedirs(OUTPUT_DIR) log = Logger("%s/credmap" % OUTPUT_DIR) log.open() def get_targets(): """ Retrieve and yield list of sites (targets) for testing. """ for site in sites: _ = populate_site(site, args) if not _: continue target = Website(_, {"verbose": args.verbose}) if not target.user_agent: target.user_agent = args.user_agent yield target def login(): """ Verify credentials for login and check if login was successful. """ if (target.username_or_email == "email" and not credentials["email"] or target.username_or_email == "username" and not credentials["username"]): if args.verbose: print( "%s Skipping %s\"%s\" since " "no \"%s\" was specified.\n" % (INFO, "[%s:%s] on " % (credentials["username"] or credentials["email"], credentials["password"]) if args.load_file else "", color(target.name), color(target.username_or_email, BW))) login_skipped.append(target.name) return print("%s Testing %s\"%s\"..." % (TEST, "[%s:%s] on " % (credentials["username"] or credentials["email"], credentials["password"]) if args.load_file else "", color(target.name, BW))) cookie_handler.clear() if target.perform_login(credentials, cookie_handler): log.write(">>> %s - %s:%s\n" % (target.name, credentials["username"] or credentials["email"], credentials["password"])) login_sucessful.append( "%s%s" % (target.name, " [%s:%s]" % (credentials["username"] or credentials["email"], credentials["password"]) if args.load_file else "")) else: login_failed.append(target.name) if args.load_file: if args.cred_format: separators = [ re.escape(args.cred_format[1]), re.escape(args.cred_format[3]) if len(args.cred_format) > 3 else "\n" ] cred_format = re.match(r"(u|e|p)[^upe](u|e|p)(?:[^upe](u|e|p))?", args.cred_format) if not cred_format: print("%s Could not parse --format: \"%s\"" % (ERROR, color(args.cred_format, BW))) exit() cred_format = [ v.replace("e", "email").replace("u", "username").replace( "p", "password") for v in cred_format.groups() if v is not None ] with open(args.load_file, "r") as load_list: for user in load_list: if args.cred_format: match = re.match( r"([^{0}]+){0}([^{1}]+)(?:{1}([^\n]+))?".format( separators[0], separators[1]), user) credentials = dict(zip(cred_format, match.groups())) credentials["password"] = quote(credentials["password"]) if ("email" in credentials and not re.match( r"^[A-Za-z0-9._%+-]+@(?:[A-Z" r"a-z0-9-]+\.)+[A-Za-z]{2,12}$", credentials["email"])): print("%s Specified e-mail \"%s\" does not appear " "to be correct. Skipping...\n" % (WARN, color(credentials["email"], BW))) continue if "email" not in credentials: credentials["email"] = None elif "username" not in credentials: credentials["username"] = None else: user = user.rstrip().split(":", 1) if not user[0]: if args.verbose: print("%s Could not parse credentials: \"%s\"\n" % (WARN, color(user, BW))) continue match = re.match( r"^[A-Za-z0-9._%+-]+@(?:[A-Z" r"a-z0-9-]+\.)+[A-Za-z]{2,12}$", user[0]) credentials = { "email": user[0] if match else None, "username": None if match else user[0], "password": quote(user[1]) } for target in get_targets(): login() else: credentials = { "username": args.username, "email": args.email, "password": quote(args.password) } for target in get_targets(): login() log.close() if not args.verbose: print() if len(login_sucessful) > 0 or len(login_failed) > 0: _ = "%s/%s" % (color(len(login_sucessful), BW), color(len(login_sucessful) + len(login_failed), BW)) sign = PLUS if len(login_sucessful) > (len(login_failed) + len(login_skipped)) else INFO print( "%s Succesfully logged in%s." % (sign, " with %s credentials on the list." % _ if args.load_file else "to %s websites." % _), ) print("%s An overall success rate of %s.\n" % (sign, color( "%%%s" % (100 * len(login_sucessful) / (len(login_sucessful) + len(login_failed))), BW))) if len(login_sucessful) > 0: print("%s The provided credentials worked on the following website%s: " "%s\n" % (PLUS, "s" if len(login_sucessful) != 1 else "", ", ".join(login_sucessful))) print("%s Finished tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW)))
def run(opt): torch.manual_seed(opt.seed) torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test print('Setting up data...') Dataset = get_dataset(opt.dataset, opt.task, opt.multi_scale) # if opt.task==mot -> JointDataset f = open(opt.data_cfg ) # choose which dataset to train '../src/lib/cfg/mot15.json', data_config = json.load(f) trainset_paths = data_config['train'] # 训练集路径 dataset_root = data_config['root'] # 数据集所在目录 print("Dataset root: %s" % dataset_root) f.close() # Image data transformations transforms = T.Compose([T.ToTensor()]) # Dataset dataset = Dataset(opt=opt, root=dataset_root, paths=trainset_paths, img_size=opt.input_wh, augment=True, transforms=transforms) opt = opts().update_dataset_info_and_set_heads(opt, dataset) print("opt:\n", opt) logger = Logger(opt) # os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str # 多GPU训练 # print("opt.gpus_str: ", opt.gpus_str) opt.device = torch.device('cuda:0' if opt.gpus[0] >= 0 else 'cpu') # 设置GPU #opt.device = device #NC UPDATE - fallback to original fairmot #opt.gpus = my_visible_devs #NC UPDATE print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) # 初始化优化器 optimizer = torch.optim.Adam(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) # Get dataloader if opt.is_debug: if opt.multi_scale: train_loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=opt.batch_size, shuffle=False, pin_memory=True, drop_last=True) # debug时不设置线程数(即默认为0) else: train_loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=opt.batch_size, shuffle=True, pin_memory=True, drop_last=True) # debug时不设置线程数(即默认为0) else: if opt.multi_scale: train_loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=opt.batch_size, shuffle=False, num_workers=opt.num_workers, pin_memory=True, drop_last=True) else: train_loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=opt.batch_size, shuffle=True, pin_memory=True, drop_last=True) # debug时不设置线程数(即默认为0) print('Starting training...') Trainer = train_factory[opt.task] trainer = Trainer(opt=opt, model=model, optimizer=optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) # trainer.set_device(opt.gpus, opt.chunk_sizes, device) #NC UPDATE best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' # Train an epoch log_dict_train, _ = trainer.train(epoch, train_loader) logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) else: # mcmot_last_track or mcmot_last_det if opt.id_weight > 0: # do tracking(detection and re-id) save_model( os.path.join(opt.save_dir, 'mcmot_last_track_' + opt.arch + '.pth'), epoch, model, optimizer) else: # only do detection # save_model(os.path.join(opt.save_dir, 'mcmot_last_det_' + opt.arch + '.pth'), # epoch, model, optimizer) save_model( os.path.join(opt.save_dir, 'mcmot_last_det_' + opt.arch + '.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr if epoch % 10 == 0: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) logger.close()
def main(opt): torch.manual_seed(opt.seed) torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test Dataset = get_dataset(opt.dataset, opt.task) opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) logger = Logger(opt) os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) optimizer = torch.optim.Adam(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) Trainer = train_factory[opt.task] trainer = Trainer(opt, model, optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) print('Setting up data...') val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'), batch_size=1, shuffle=False, num_workers=0, pin_memory=True) if opt.test: _, preds = trainer.val(0, val_loader) val_loader.dataset.run_eval(preds, opt.save_dir) return train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'), batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) print('Starting training...') best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' log_dict_train, _ = trainer.train(epoch, train_loader) logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) with torch.no_grad(): log_dict_val, preds = trainer.val(epoch, val_loader) for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if log_dict_val[opt.metric] < best: best = log_dict_val[opt.metric] save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch, model) else: save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr logger.close()
def main(opt, opt_t): torch.manual_seed(opt.seed) torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test if opt.target_dataset: Dataset_target = get_dataset(opt_t.target_dataset, opt_t.task) opt_t = opts().update_dataset_info_and_set_heads( opt_t, Dataset_target) # target dataset Dataset_source = get_dataset(opt.source_dataset, opt.task) opt = opts().update_dataset_info_and_set_heads( opt, Dataset_source) # source dataset print(opt) logger = Logger(opt) # record os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) # create model optimizer = torch.optim.Adam(model.parameters(), opt.lr) # create optimizer start_epoch = 0 if opt.load_model != '': # load model model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) Trainer = train_factory[opt.task] # set trainer function trainer = Trainer(opt, model, optimizer) # initial trainer trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) print('Setting up source_val data...') val_source_loader = torch.utils.data.DataLoader(Dataset_source(opt, 'val'), batch_size=1, shuffle=False, num_workers=0, pin_memory=True) if opt.test: _, preds = trainer.val(0, val_source_loader) val_source_loader.dataset.run_eval(preds, opt.save_dir) return # source loader print('Setting up source_train data...') train_source_loader = torch.utils.data.DataLoader( Dataset_source(opt, 'train'), # modify SOURCE dataset parameters batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) if opt.target_dataset: # target loader print('Setting up target_train data...') train_target_loader = torch.utils.data.DataLoader( Dataset_target(opt_t, 'train'), # modify TARGET dataset parameters batch_size=opt_t.batch_size, shuffle=True, num_workers=opt_t.num_workers, pin_memory=True, drop_last=True) print('DA MODE') else: train_target_loader = None print('Starting training...') best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' log_dict_train, _ = trainer.train(epoch, train_source_loader, train_target_loader) # do train logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): # log information logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) # if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: # save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), # save last-model # epoch, model, optimizer) # with torch.no_grad(): # log_dict_val, preds = trainer.val(epoch, val_source_loader) # cal val-set loss # for k, v in log_dict_val.items(): # logger.scalar_summary('val_{}'.format(k), v, epoch) # logger.write('{} {:8f} | '.format(k, v)) # if log_dict_val[opt.metric] < best: # best = log_dict_val[opt.metric] # save_model(os.path.join(opt.save_dir, 'model_best.pth'), # save best-model # epoch, model) # else: save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: # update learning rate save_model( os.path.join( opt.save_dir, 'model_{}.pth'.format(epoch)), # save lr_step-model epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr logger.close()
def main(opt): if opt.disable_cudnn: torch.backends.cudnn.enabled = False print('Cudnn is disabled.') logger = Logger(opt) opt.device = torch.device('cuda:{}'.format(opt.gpus[0])) Dataset = dataset_factory[opt.dataset] train, val = task_factory[opt.task] model, optimizer, start_epoch = create_model(opt) if len(opt.gpus) > 1: model = torch.nn.DataParallel(model, device_ids=opt.gpus).cuda(opt.device) else: model = model.cuda(opt.device) val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'), batch_size=1, shuffle=False, num_workers=1, pin_memory=True) if opt.test: log_dict_train, preds = val(0, opt, val_loader, model) sio.savemat(os.path.join(opt.save_dir, 'preds.mat'), mdict={'preds': preds}) return train_loader = torch.utils.data.DataLoader( Dataset(opt, 'train'), batch_size=opt.batch_size * len(opt.gpus), shuffle=True, # if opt.debug == 0 else False, num_workers=opt.num_workers, pin_memory=True) best = -1 for epoch in range(start_epoch, opt.num_epochs + 1): mark = epoch if opt.save_all_models else 'last' log_dict_train, _ = train(epoch, opt, train_loader, model, optimizer) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) log_dict_val, preds = val(epoch, opt, val_loader, model) for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if log_dict_val[opt.metric] > best: best = log_dict_val[opt.metric] save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch, model) else: save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr logger.close()
def main(opt): torch.manual_seed(opt.seed) # benchmark=True 自动寻找最适合当前配置的高效算法,来达到优化运行效率的wento torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test Dataset = get_dataset(opt.dataset, opt.task) opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) logger = Logger(opt) def adjust_learning_rate(optimizer, epoch): # use warmup if epoch < 5: lr = opt.lr * ((epoch + 1) / 5) else: # use cosine lr PI = 3.14159 lr = opt.lr * 0.5 * (1 + math.cos(epoch * PI / 20)) # print(1111) for param_group in optimizer.param_groups: param_group['lr'] = lr os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) # optimizer = torch.optim.Adam(model.parameters(), opt.lr) optimizer = torch.optim.SGD(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) Trainer = train_factory[opt.task] trainer = Trainer(opt, model, optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) print('Setting up data...') # num_worker=0是主进程读取; >0使用多进程读取,子进程读取数据时,训练程序会卡住,GPU utils为0, val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'), batch_size=1, shuffle=False, num_workers=1, pin_memory=True) if opt.test: _, preds = trainer.val(0, val_loader) val_loader.dataset.run_eval(preds, opt.save_dir) return train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'), batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) print('Starting training...') best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): adjust_learning_rate(optimizer, epoch) mark = epoch if opt.save_all else 'last' log_dict_train, _ = trainer.train(epoch, train_loader) logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % 2 == 0 and epoch > 10: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) with torch.no_grad(): log_dict_val, preds = trainer.val(epoch, val_loader) for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if log_dict_val[opt.metric] < best: best = log_dict_val[opt.metric] save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch, model) # test(opt) # opt.model = None if epoch > 20: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) # elif 80 < epoch <=100 and epoch % 3 == 0: # save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), # epoch, model, optimizer) logger.write('\n') # if epoch in opt.lr_step: # save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), # epoch, model, optimizer) # lr = opt.lr * (0.1 ** (opt.lr_step.index(epoch) + 1)) # print('Drop LR to', lr) # for param_group in optimizer.param_groups: # param_group['lr'] = lr print('Epoch is Finished') logger.close()
def main(): """ Initializes and executes the program. """ login_sucessful = [] login_failed = [] login_skipped = [] version = check_revision(VERSION) print("%s\n\n%s %s (%s)\n" % ( BANNER % tuple([color(_) for _ in BANNER_PASSWORDS]), NAME, version, URL)) args = parse_args() if args.update: update() exit() sites = list_sites() if args.list: for _ in sites: print("- %s" % _) exit() if not args.password and not args.load_file: args.password = getpass("%s Please enter password:"******"(?P<type>[^:]+)://(?P<address>[^:]+)" r":(?P<port>\d+)", args.proxy, re.I) if match: if match.group("type").upper() in ("HTTP", "HTTPS"): proxy_host = "%s:%s" % (match.group("address"), match.group("port")) proxy_handler = ProxyHandler({"http": proxy_host, "https": proxy_host}) else: from thirdparty.socks import socks if match.group("type").upper() == "SOCKS4": socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS4, match.group("address"), int(match.group("port")), True) elif match.group("type").upper() == "SOCKS5": socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, match.group("address"), int(match.group("port")), True) proxy_handler = None else: proxy_handler = ProxyHandler() else: proxy_handler = None opener = build_opener(HTTPHandler(), HTTPSHandler(), HTTPCookieProcessor(cookie_handler)) if proxy_handler: opener.add_handler(proxy_handler) install_opener(opener) with open(USER_AGENTS_FILE, 'r') as ua_file: args.user_agent = sample(ua_file.readlines(), 1)[0].strip() if args.only: sites = [site for site in sites if site in args.only] elif args.exclude: sites = [site for site in sites if site not in args.exclude] print("%s Loaded %d %s to test." % (INFO, len(sites), "site" if len(sites) == 1 else "sites")) if args.load_file: if not isfile(args.load_file): print("%s could not find the file \"%s\"" % (WARN, color(args.load_file))) exit() _ = sum(1 for line in open(args.load_file, "r")) if _ < 1: print("%s the file \"%s\" doesn't contain any valid credentials." % (WARN, color(args.load_file))) exit() print("%s Loaded %d credential%s from \"%s\".\n" % (INFO, _, "s" if _ != 1 else "", color(args.load_file))) print("%s Starting tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW))) if not exists(OUTPUT_DIR): makedirs(OUTPUT_DIR) log = Logger("%s/credmap" % OUTPUT_DIR) log.open() def get_targets(): """ Retrieve and yield list of sites (targets) for testing. """ for site in sites: _ = populate_site(site, args) if not _: continue target = Website(_, {"verbose": args.verbose}) if not target.user_agent: target.user_agent = args.user_agent yield target def login(): """ Verify credentials for login and check if login was successful. """ if(target.username_or_email == "email" and not credentials["email"] or target.username_or_email == "username" and not credentials["username"]): if args.verbose: print("%s Skipping %s\"%s\" since " "no \"%s\" was specified.\n" % (INFO, "[%s:%s] on " % (credentials["username"] or credentials["email"], credentials["password"]) if args.load_file else "", color(target.name), color(target.username_or_email, BW))) login_skipped.append(target.name) return print("%s Testing %s\"%s\"..." % (TEST, "[%s:%s] on " % (credentials["username"] or credentials["email"], credentials["password"]) if args.load_file else "", color(target.name, BW))) cookie_handler.clear() if target.perform_login(credentials, cookie_handler): log.write(">>> %s - %s:%s\n" % (target.name, credentials["username"] or credentials["email"], credentials["password"])) login_sucessful.append("%s%s" % (target.name, " [%s:%s]" % (credentials["username"] or credentials["email"], credentials["password"]) if args.load_file else "")) else: login_failed.append(target.name) if args.load_file: if args.cred_format: separators = [re.escape(args.cred_format[1]), re.escape(args.cred_format[3]) if len(args.cred_format) > 3 else "\n"] cred_format = re.match(r"(u|e|p)[^upe](u|e|p)(?:[^upe](u|e|p))?", args.cred_format) if not cred_format: print("%s Could not parse --format: \"%s\"" % (ERROR, color(args.cred_format, BW))) exit() cred_format = [v.replace("e", "email") .replace("u", "username") .replace("p", "password") for v in cred_format.groups() if v is not None] with open(args.load_file, "r") as load_list: for user in load_list: if args.cred_format: match = re.match(r"([^{0}]+){0}([^{1}]+)(?:{1}([^\n]+))?" .format(separators[0], separators[1]), user) credentials = dict(zip(cred_format, match.groups())) credentials["password"] = quote( credentials["password"]) if("email" in credentials and not re.match(r"^[A-Za-z0-9._%+-]+@(?:[A-Z" r"a-z0-9-]+\.)+[A-Za-z]{2,12}$", credentials["email"])): print("%s Specified e-mail \"%s\" does not appear " "to be correct. Skipping...\n" % (WARN, color( credentials["email"], BW))) continue if "email" not in credentials: credentials["email"] = None elif "username" not in credentials: credentials["username"] = None else: user = user.rstrip().split(":", 1) if not user[0]: if args.verbose: print("%s Could not parse credentials: \"%s\"\n" % (WARN, color(user, BW))) continue match = re.match(r"^[A-Za-z0-9._%+-]+@(?:[A-Z" r"a-z0-9-]+\.)+[A-Za-z]{2,12}$", user[0]) credentials = {"email": user[0] if match else None, "username": None if match else user[0], "password": quote(user[1])} for target in get_targets(): login() else: credentials = {"username": args.username, "email": args.email, "password": quote(args.password)} for target in get_targets(): login() log.close() if not args.verbose: print() if len(login_sucessful) > 0 or len(login_failed) > 0: _ = "%s/%s" % (color(len(login_sucessful), BW), color(len(login_sucessful) + len(login_failed), BW)) sign = PLUS if len(login_sucessful) > (len(login_failed) + len(login_skipped)) else INFO print("%s Succesfully logged in%s." % (sign, " with %s credentials on the list." % _ if args.load_file else "to %s websites." % _),) print("%s An overall success rate of %s.\n" % (sign, color("%%%s" % (100 * len(login_sucessful) / (len(login_sucessful) + len(login_failed))), BW))) if len(login_sucessful) > 0: print("%s The provided credentials worked on the following website%s: " "%s\n" % (PLUS, "s" if len(login_sucessful) != 1 else "", ", ".join(login_sucessful))) print("%s Finished tests at: \"%s\"\n" % (INFO, color(strftime("%X"), BW)))
def main(opt): torch.manual_seed( opt.seed ) # opt.seed: default=317 ;加上torch.manual_seed这个函数调用的话,打印出来的随机数每次都一样。 torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test Dataset = get_dataset( opt.dataset, opt.task ) # opt.dataset = coco, opt.task = ctdet (| ddd | multi_pose | exdet) opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) logger = Logger(opt) os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) optimizer = torch.optim.Adam(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) Trainer = train_factory[opt.task] trainer = Trainer(opt, model, optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) print('Setting up data...') val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'), batch_size=1, shuffle=False, num_workers=1, pin_memory=True) # modified by zy # val_loader = torch.utils.data.DataLoader(Dataset(opt, 'test'), batch_size=1, shuffle=False, num_workers=1,pin_memory=True) if opt.test: _, preds = trainer.val(0, val_loader) val_loader.dataset.run_eval(preds, opt.save_dir) return train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'), batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) output_choice_log = '/home/zy/zy/2new_network/CenterNet-master/output_choice.log' if os.path.exists(output_choice_log): os.remove(output_choice_log) print('Starting training...') best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' try: log_dict_train, _ = trainer.train( epoch, train_loader ) # !!!!!!!! train = self.run_epoch('train', epoch, data_loader) except Exception as e: # 如果发生异常,那就返回预设的loss值 print('Error_train!!!', e) print(traceback.format_exc()) continue logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) with torch.no_grad(): try: log_dict_val, preds = trainer.val(epoch, val_loader) except Exception as e: # 如果发生异常,那就返回预设的loss值 print('Error_train!!!', e) print(traceback.format_exc()) continue for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if log_dict_val[opt.metric] < best: best = log_dict_val[opt.metric] save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch, model) else: save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr logger.close()
def main(opt): torch.manual_seed(opt.seed) torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test print('Setting up data...') Dataset = get_dataset(opt.task) f = open(opt.data_cfg) data_config = json.load(f) trainset_paths = data_config['train'] dataset_root = data_config['root'] f.close() transforms = T.Compose([T.ToTensor()]) dataset = Dataset(opt, dataset_root, trainset_paths, (1088, 608), augment=True, transforms=transforms) opt = opts().update_dataset_info_and_set_heads(opt, dataset) print(opt) logger = Logger(opt) os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) optimizer = torch.optim.Adam(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) # Get dataloader train_loader = torch.utils.data.DataLoader(dataset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) print('Starting training...') Trainer = train_factory[opt.task] trainer = Trainer(opt, model, optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' log_dict_train, _ = trainer.train(epoch, train_loader) logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) else: save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr if epoch % 5 == 0: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) logger.close()