def __init__(self, AceConfig, AceProxy): self.AceConfig = AceConfig self.picons = self.channels = self.playlist = self.etag = self.last_modified = None self.playlisttime = gevent.time.time() self.headers = {'User-Agent': 'Magic Browser'} if config.updateevery: schedule(config.updateevery * 60, self.Playlistparser)
def __init__(self, AceConfig, AceProxy): self.config = AceConfig self.logger = logging.getLogger('plugin_TorrentFilms') self.playlist = [] self.videoextdefaults = ('.3gp', '.aac', '.ape', '.asf', '.avi', '.dv', '.divx', '.flac', '.flc', '.flv', '.m2ts', '.m4a', '.mka', '.mkv', '.mpeg', '.mpeg4', '.mpegts', '.mpg4', '.mp3', '.mp4', '.mpg', '.mov', '.m4v', '.ogg', '.ogm', '.ogv', '.oga', '.ogx', '.qt', '.rm', '.swf', '.ts', '.vob', '.wmv', '.wav', '.webm') if config.updateevery: schedule(config.updateevery * 60, self.playlistdata)
def download_video(self, video_lists): print(">>视频列表下载开始...") print(video_lists) x = 0 file_path = self.file_path if not os.path.exists(file_path): os.mkdir(file_path) for video in video_lists: video_name = video[0] video_url = video[1] # 获取真实地址 print("解析地址中...") real_rul = self.get_real_url(video_url, 1) # print("真实地址"+real_rul) # 现在时间戳 now_time = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') # 文件名为当前时间戳+视频title file_name = str(now_time) + '_' + video_name print(file_name + '开始下载...') request.urlretrieve(real_rul, file_path + '%s.mp4' % file_name, reporthook=utils.schedule()) print(file_name + '下载完成...') x = x + 1 print('下载完成...共' + str(x) + '个视频') print('视频保存地址:' + file_path)
def init_callbacks(self, ): scheduler = None tensorboard_callback = None self.callbacks = [] if (self.lr_decay_policy == 'poly'): scheduler = schedule( self.lr, self.lr_decay_power, self.epochs ) #PolynomialDecay(maxEpochs=100, initAlpha=0.01, power=self.lr_decay_power) # initializing tensorboard # shutil.rmtree(self.logdir+"/train/") os.makedirs(self.logdir, exist_ok=True) tensorboard_callback = TensorBoard(log_dir=self.logdir) if (scheduler): print("learning rate scheduler added\n\n") self.callbacks.append(LearningRateScheduler(scheduler)) if (tensorboard_callback): print("tensor boarda callback added with folder " + self.logdir + "\n\n") self.callbacks.append(tensorboard_callback) # Create a callback that saves the model's weights every 5 epochs cp_callback = tf.keras.callbacks.ModelCheckpoint( filepath="/home/essys/projects/segmentation/checkpoints/" + self.train_id + "/cp-{epoch:04d}.ckpt", verbose=0, save_weights_only=True, period=50) self.callbacks.append(cp_callback) diagnoser = ModelDiagonoser(self.training_generator, self.batch_size, self.n_samples, self.logdir, self.input_shape, self.n_classes) self.callbacks.append(diagnoser)
def swa_train(model, swa_model, train_iter, valid_iter, optimizer, criterion, pretrain_epochs, swa_epochs, swa_lr, cycle_length, device, writer, cpt_filename): swa_n = 1 swa_model.load_state_dict(copy.deepcopy(model.state_dict())) utils.save_checkpoint( cpt_directory, 1, '{}-swa-{:2.4f}-{:03d}-{}'.format(date, swa_lr, cycle_length, cpt_filename), state_dict=model.state_dict(), swa_state_dict=swa_model.state_dict(), swa_n=swa_n, optimizer=optimizer.state_dict() ) for e in range(swa_epochs): epoch = e + pretrain_epochs time_ep = time.time() lr = utils.schedule(epoch, cycle_length, lr_init, swa_lr) utils.adjust_learning_rate(optimizer, lr) train_res = utils.train_epoch(model, train_iter, optimizer, criterion, device) valid_res = utils.evaluate(model, valid_iter, criterion, device) utils.moving_average(swa_model, model, swa_n) swa_n += 1 utils.bn_update(train_iter, swa_model) swa_res = utils.evaluate(swa_model, valid_iter, criterion, device) time_ep = time.time() - time_ep values = [epoch + 1, lr, swa_lr, cycle_length, train_res['loss'], valid_res['loss'], swa_res['loss'], None, None, time_ep] writer.writerow(values) table = tabulate.tabulate([values], columns, tablefmt='simple', floatfmt='8.4f') if epoch % 20 == 0: table = table.split('\n') table = '\n'.join([table[1]] + table) else: table = table.split('\n')[2] print(table) utils.save_checkpoint( cpt_directory, epoch + 1, '{}-swa-{:2.4f}-{:03d}-{}'.format(date, swa_lr, cycle_length, cpt_filename), state_dict=model.state_dict(), swa_state_dict=swa_model.state_dict(), swa_n=swa_n, optimizer=optimizer.state_dict() )
def train_network(self): for t in range(self.T): lr = schedule(self.lr0, self.d0, t) # data, labels = shuffle(self.train, self.labels) data = self.train labels = self.labels # for ex in range(self.N): for ex in range(1): features = data[ex, :] lab = labels[ex] print(features, lab) self.ff(features) self.bp(features, lab) for layer in self.layers: if layer.layer_num == self.d: layer.ws -= lr * layer.dws.T else: layer.ws -= lr * layer.dws return self
def train_and_apply(self, train, test): train_err = np.zeros(self.T) test_err = np.zeros(self.T) for t in range(self.T): lr = schedule(self.lr0, self.d0, t) data, labels = shuffle(self.train, self.labels) for ex in range(self.N): features = data[ex, :] lab = labels[ex] self.ff(features) self.bp(features, lab) for layer in self.layers: if layer.layer_type == 'out': layer.ws -= lr * layer.dws.T else: layer.ws -= lr * layer.dws err = self.apply_network(train) train_err[t] = err err = self.apply_network(test) test_err[t] = err return self, train_err, test_err
def print_pretty_matrix(matrix): col_labels = ['slot/day', 'Mon', 'Tu', 'Wed', 'Thur', 'Fri'] table_vals = [[schedule()["pairs"][i], '', '', '', '', ''] for i in range(6)] table = prettytable.PrettyTable(col_labels, hrules=prettytable.ALL) for i in range(len(matrix)): for j in range(len(matrix[i])): text = '' teachers = set() for c in matrix[i][j]: teachers.add(c.subject.teacher) text += '[subject: {}; tutor: {}; room: {}]\n'.format( c.subject.__repr__(), c.subject.teacher, c.classroom) table_vals[j][i + 1] = text if len(teachers) < len(matrix[i][j]): print("error") for row in table_vals: table.add_row(row) print(table)
else: logger.error('Cannot drop privileges to user %s' % AceConfig.aceproxyuser) sys.exit(1) # Creating ClientCounter AceProxy.pool = Pool() AceProxy.clientcounter = ClientCounter() #### AceEngine startup AceProxy.ace = findProcess('ace_engine.exe' if AceConfig.osplatform == 'Windows' else os.path.basename(AceConfig.acecmd)) if not AceProxy.ace and AceConfig.acespawn: if spawnAce(): logger.info('Local AceStream engine spawned with pid %s' % AceProxy.ace.pid) schedule(AceConfig.acestartuptimeout, checkAce) # Start AceEngine alive watchdog elif AceProxy.ace: AceProxy.ace = psutil.Process(AceProxy.ace) logger.info('Local AceStream engine found with pid %s' % AceProxy.ace.pid) # If AceEngine started (found) localy if AceProxy.ace: AceConfig.ace['aceHostIP'] = '127.0.0.1' # Refreshes the acestream.port file for OS Windows..... if AceConfig.osplatform == 'Windows': detectPort() else: gevent.sleep(AceConfig.acestartuptimeout) else: url = 'http://{aceHostIP}:{aceHTTPport}/webui/api/service'.format( **AceConfig.ace) params = { 'method': 'get_version',
# net = DPN92(args.cifar100) net = ACN(args.cifar100) if use_cuda: net.cuda() net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) cudnn.benchmark = True criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) if args.sch: scheduler = schedule(optimizer, milestones=[150, 250], gamma=0.1) # Training def train(epoch): print('\nEpoch: %d' % epoch) net.train() train_loss = 0 correct = 0 total = 0 print(net) print(vars(args)) for batch_idx, (inputs, targets) in enumerate(trainloader): if use_cuda: inputs, targets = inputs.cuda(), targets.cuda() optimizer.zero_grad()
def main(args): if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") train_loader, test_loader = load_dataset(args.label, args.batch_size, args.half_length, args.nholes) if args.label == 10: model = ShakeResNet(args.depth, args.w_base, args.label) else: model = ShakeResNeXt(args.depth, args.w_base, args.cardinary, args.label) model = torch.nn.DataParallel(model).cuda() cudnn.benckmark = True if args.optimizer == 'sgd': print("using sgd") opt = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov) elif args.optimizer == 'abd': print("using adabound") opt = abd.AdaBound(model.parameters(), lr=args.lr, gamma=args.gamma, weight_decay=args.weight_decay, final_lr=args.final_lr) elif args.optimizer == 'swa': print("using swa") opt = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) steps_per_epoch = len(train_loader.dataset) / args.batch_size steps_per_epoch = int(steps_per_epoch) opt = swa(opt, swa_start=args.swa_start * steps_per_epoch, swa_freq=steps_per_epoch, swa_lr=args.swa_lr) else: print("not valid optimizer") exit loss_func = nn.CrossEntropyLoss().cuda() headers = [ "Epoch", "LearningRate", "TrainLoss", "TestLoss", "TrainAcc.", "TestAcc." ] #if args.optimizer=='swa': # headers = headers[:-1] + ['swa_te_loss', 'swa_te_acc'] + headers[-1:] # swa_res = {'loss': None, 'accuracy': None} logger = utils.Logger(args.checkpoint, headers, mod=args.optimizer) for e in range(args.epochs): if args.optimizer == 'swa': lr = utils.schedule(e, args.optimizer, args.epochs, args.swa_start, args.swa_lr, args.lr) utils.adjust_learning_rate(opt, lr) elif args.optimizer == 'sgd': lr = utils.cosine_lr(opt, args.lr, e, args.epochs) else: exit #train train_loss, train_acc, train_n = utils.train_epoch( train_loader, model, opt) #eval test_loss, test_acc, test_n = utils.eval_epoch(test_loader, model) logger.write(e + 1, lr, train_loss / train_n, test_loss / test_n, train_acc / train_n * 100, test_acc / test_n * 100) if args.optimizer == 'swa' and ( e + 1) >= args.swa_start and args.eval_freq > 1: if e == 0 or e % args.eval_freq == args.eval_freq - 1 or e == args.epochs - 1: opt.swap_swa_sgd() opt.bn_update(train_loaders, model, device='cuda') #swa_res = utils.eval_epoch(test_loaders['test'], model) opt.swap_swa_sgd()