def __init__(self, model_factory, trainloader, valloader, params, dataset_cfg, experiment_cfg, best_metric_recorder, restore_checkpoint=None, logger=None): logger = logger if logger else Logger.get() items = model_factory(params) self.model = items['model'] self.optimizer = items['optimizer'] self.criterion = items['criterion'] self.metrics = items['metrics'] self.best_metric_recorder = best_metric_recorder self.trainloader = trainloader self.valloader = valloader self.num_epochs = params.num_epochs self.running_avg_steps = params.running_avg_steps self.checkpoint = Checkpoint( checkpoint_dir=experiment_cfg.experiment_dir(), filename=experiment_cfg.checkpoint_filename(), best_checkpoint=experiment_cfg.best_checkpoint(), latest_checkpoint=experiment_cfg.latest_checkpoint(), logger=logger) self.best_metrics_file = experiment_cfg.best_metrics_file( dataset_cfg.val_name()) self.latest_metrics_file = experiment_cfg.latest_metrics_file( dataset_cfg.val_name()) self.restore_checkpoint = restore_checkpoint self.logger = logger
def __init__(self, model_factory, dataset, params, dataset_cfg, experiment_cfg, restore_checkpoint, logger=None): items = model_factory(params) self.logger = logger if logger else Logger.get() self.model = items['model'] # self.optimizer = items['optimizer'] self.criterion = items['criterion'] self.metrics = items['metrics'] # restore model from checkpoint checkpoint = Checkpoint( checkpoint_dir=experiment_cfg.experiment_dir(), filename=experiment_cfg.checkpoint_filename(), best_checkpoint=experiment_cfg.best_checkpoint(), latest_checkpoint=experiment_cfg.latest_checkpoint(), logger=self.logger) status = checkpoint.restore(self.model, None, restore_checkpoint) assert status, "Restore model from the checkpoint: {}, failed".format( restore_checkpoint) self.dataset = dataset self.metrics_file = experiment_cfg.metrics_file( restore_checkpoint, self.dataset.dataset_name)
def __init__(self, args): self.config = args # parameters self.start_epoch = 1 self.max_epoch = args.max_epoch self.save_dir = args.save_dir self.device = args.device self.verbose = args.verbose self.max_points = args.max_points self.voxel_size = args.voxel_size self.model = args.model.to(self.device) self.optimizer = args.optimizer self.scheduler = args.scheduler self.scheduler_freq = args.scheduler_freq self.snapshot_freq = args.snapshot_freq self.snapshot_dir = args.snapshot_dir self.benchmark = args.benchmark self.iter_size = args.iter_size self.verbose_freq = args.verbose_freq self.w_circle_loss = args.w_circle_loss self.w_overlap_loss = args.w_overlap_loss self.w_saliency_loss = args.w_saliency_loss self.desc_loss = args.desc_loss self.best_loss = 1e5 self.best_recall = -1e5 self.writer = SummaryWriter(log_dir=args.tboard_dir) self.logger = Logger(args.snapshot_dir) self.logger.write( f'#parameters {sum([x.nelement() for x in self.model.parameters()])/1000000.} M\n' ) if (args.pretrain != ''): self._load_pretrain(args.pretrain) self.loader = dict() self.loader['train'] = args.train_loader self.loader['val'] = args.val_loader self.loader['test'] = args.test_loader with open(f'{args.snapshot_dir}/model', 'w') as f: f.write(str(self.model)) f.close()
parser.add_argument('--encoding', default='utf8', help="The encoding for input and output file.") args = parser.parse_args() dataset_cfg = DatasetCfg(args.data_dir) exp_cfg = ExperimentCfg(args.exp_dir) inputs_file = args.inputs_file outputs_file = args.outputs_file restore_checkpoint = args.restore_checkpoint encoding = args.encoding msg = "Inputs file not exists: {}" assert os.path.isfile(inputs_file), msg.format(inputs_file) logger = Logger.set(os.path.join(exp_cfg.experiment_dir(), 'predict.log')) checkpoint = Checkpoint( checkpoint_dir=exp_cfg.experiment_dir(), filename=exp_cfg.checkpoint_filename(), best_checkpoint=exp_cfg.best_checkpoint(), latest_checkpoint=exp_cfg.latest_checkpoint(), logger=logger) # load params word_vocab = Vocab(words_file) tag_vocab = Vocab(tags_file) params = Params(exp_cfg.params_file()) params.update(Params(dataset_cfg.params_file())) params.set('cuda', torch.cuda.is_available())
epochs=args.epochs) elif args.i_what == 'iFlow': metadata.update({"device": device}) model = iFlow(args=metadata).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, \ factor=args.lr_drop_factor, \ patience=args.lr_patience, \ verbose=True) # factor=0.1 and patience=4 ste = time.time() print('setup time: {}s'.format(ste - st)) # setup loggers logger = Logger(logdir=LOG_FOLDER) # 'log/' exp_id = logger.get_id() # 1 tensorboard_run_name = TENSORBOARD_RUN_FOLDER + 'exp' + str( exp_id) + '_'.join( map(str, [ '', args.batch_size, args.max_iter, args.lr, args.hidden_dim, args.depth, args.anneal ])) # 'runs/exp1_64_12500_0.001_50_3_False' writer = SummaryWriter(logdir=tensorboard_run_name) if args.i_what == 'iFlow': logger.add('log_normalizer') logger.add('neg_log_det')
def logger_instance(D_learnrate, G_learnrate, Dataset_name): return Logger(model_name='GAN D_LR=' + str(D_learnrate) + ' G_LR' + str(G_learnrate), data_name=Dataset_name)
import argparse import os import align.detect_face as detect_face import cv2 import numpy as np import tensorflow as tf from lib.face_utils import judge_side_face from lib.utils import Logger, mkdir from src.sort import Sort logger = Logger(__name__) def main(): global colours, img_size args = parse_args() #一个或多个视频存放路径 root_dir = args.root_dir #采集并裁剪人脸保存路径 output_path = args.output_path display = args.display mkdir(output_path) if display: colours = np.random.rand(32, 3) #初始化tracker tracker = Sort() logger.info('start track and extract......') with tf.Graph().as_default(): with tf.Session(
def main(args): # fix random seeds if args.seed: torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) np.random.seed(args.seed) # CNN if args.verbose: print('Architecture: {}'.format(args.arch)) model = models.__dict__[args.arch](sobel=args.sobel, dropout=args.dropout) fd = int(model.top_layer.weight.size()[1]) model.top_layer = None model.features = torch.nn.DataParallel(model.features) model.cuda() cudnn.benchmark = True # create optimizer optimizer = torch.optim.SGD( filter(lambda x: x.requires_grad, model.parameters()), lr=args.learning_rate, momentum=args.momentum, weight_decay=10**args.weight_decay, ) # define loss function criterion = nn.CrossEntropyLoss().cuda() restore(model, args.resume) # creating checkpoint repo exp_check = os.path.join(args.experiment, 'checkpoints') if not os.path.isdir(exp_check): os.makedirs(exp_check) # creating cluster assignments log cluster_log = Logger(os.path.join(args.experiment, 'clusters')) # preprocessing of data normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) tra = [ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize ] # load the data end = time.time() dataset = datasets.ImageFolder(args.data, transform=transforms.Compose(tra)) if args.verbose: print('Load dataset: {0:.2f} s'.format(time.time() - end)) dataloader = torch.utils.data.DataLoader(dataset, batch_size=args.batch, num_workers=args.workers, pin_memory=True) algs = { 'KMeans': clustering.KMeans, 'PIC': clustering.PIC, } cluster_alg = algs[args.cluster_alg](args.nmb_cluster) # training convnet with cluster_alg for epoch in range(args.start_epoch, args.epochs): end = time.time() # remove head model.top_layer = None model.classifier = nn.Sequential( *list(model.classifier.children())[:-1]) # get the features for the whole dataset features = compute_features(dataloader, model, len(dataset), args.batch) # cluster the features if args.verbose: print('Cluster the features') clustering_loss = cluster_alg.cluster(features, verbose=args.verbose) # assign pseudo-labels if args.verbose: print('Assign real labels') # train_dataset = cluster_assign(cluster_alg.images_lists, # dataset.imgs) train_dataset = cluster_assign_with_original_labels(dataset.imgs) # uniformly sample per target sampler = UnifLabelSampler(int(args.reassign * len(train_dataset)), cluster_alg.images_lists) train_dataloader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch, num_workers=args.workers, sampler=sampler, pin_memory=True, ) # set last fully connected layer mlp = list(model.classifier.children()) mlp.append(nn.ReLU(inplace=True).cuda()) model.classifier = nn.Sequential(*mlp) model.top_layer = nn.Linear(fd, len(cluster_alg.images_lists)) model.top_layer.weight.data.normal_(0, 0.01) model.top_layer.bias.data.zero_() model.top_layer.cuda() # train network with clusters as pseudo-labels end = time.time() for x in range(1000): loss = train(train_dataloader, model, criterion, optimizer, epoch) # print log if args.verbose: print('###### Epoch [{0}] ###### \n' 'Time: {1:.3f} s\n' 'Clustering loss: {2:.3f} \n' 'ConvNet loss: {3:.3f}'.format(epoch, time.time() - end, clustering_loss, loss)) try: nmi = normalized_mutual_info_score( arrange_clustering(cluster_alg.images_lists), arrange_clustering(cluster_log.data[-1])) print('NMI against previous assignment: {0:.3f}'.format(nmi)) except IndexError: pass print('####################### \n') # save running checkpoint torch.save( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict() }, os.path.join(args.experiment, 'checkpoint.pth.tar')) # save cluster assignments cluster_log.log(cluster_alg.images_lists)
if __name__ == '__main__': dataset_cfg = DatasetCfg(config.data_dir) exp_cfg = ExperimentCfg(config.base_model_dir) parser = get_parser(data_dir=dataset_cfg.data_dir(), exp_dir=exp_cfg.experiment_dir(), restore_checkpoint=exp_cfg.best_checkpoint(), dataset_name=dataset_cfg.test_name()) args = parser.parse_args() dataset_name = args.dataset_name restore_checkpoint = args.restore_checkpoint dataset_cfg.set_data_dir(args.data_dir) exp_cfg.set_experiment_dir(args.exp_dir) # set logger logger = Logger.set(exp_cfg.evaluate_log()) # load model configuration logger.info("Loading the experiment configurations...") params = Params(exp_cfg.params_file()) # cuda flag params.set('cuda', torch.cuda.is_available()) logger.info("- done.") # load datesets logger.info("Loading the {} dataset...".format(dataset_name)) # add datasets parameters into params params.update(Params(dataset_cfg.params_file())) dataset = load_data(params, dataset_cfg.data_dir(), dataset_name, params['{}_size'.format(dataset_name)]) logger.info("- done.")
class Trainer(object): def __init__(self, args): self.config = args # parameters self.start_epoch = 1 self.max_epoch = args.max_epoch self.save_dir = args.save_dir self.device = args.device self.verbose = args.verbose self.max_points = args.max_points self.voxel_size = args.voxel_size self.model = args.model.to(self.device) self.optimizer = args.optimizer self.scheduler = args.scheduler self.scheduler_freq = args.scheduler_freq self.snapshot_freq = args.snapshot_freq self.snapshot_dir = args.snapshot_dir self.benchmark = args.benchmark self.iter_size = args.iter_size self.verbose_freq = args.verbose_freq self.w_circle_loss = args.w_circle_loss self.w_overlap_loss = args.w_overlap_loss self.w_saliency_loss = args.w_saliency_loss self.desc_loss = args.desc_loss self.best_loss = 1e5 self.best_recall = -1e5 self.writer = SummaryWriter(log_dir=args.tboard_dir) self.logger = Logger(args.snapshot_dir) self.logger.write( f'#parameters {sum([x.nelement() for x in self.model.parameters()])/1000000.} M\n' ) if (args.pretrain != ''): self._load_pretrain(args.pretrain) self.loader = dict() self.loader['train'] = args.train_loader self.loader['val'] = args.val_loader self.loader['test'] = args.test_loader with open(f'{args.snapshot_dir}/model', 'w') as f: f.write(str(self.model)) f.close() def _snapshot(self, epoch, name=None): state = { 'epoch': epoch, 'state_dict': self.model.state_dict(), 'optimizer': self.optimizer.state_dict(), 'scheduler': self.scheduler.state_dict(), 'best_loss': self.best_loss, 'best_recall': self.best_recall } if name is None: filename = os.path.join(self.save_dir, f'model_{epoch}.pth') else: filename = os.path.join(self.save_dir, f'model_{name}.pth') self.logger.write(f"Save model to {filename}\n") torch.save(state, filename) def _load_pretrain(self, resume): if os.path.isfile(resume): state = torch.load(resume) self.model.load_state_dict(state['state_dict']) self.start_epoch = state['epoch'] self.scheduler.load_state_dict(state['scheduler']) self.optimizer.load_state_dict(state['optimizer']) self.best_loss = state['best_loss'] self.best_recall = state['best_recall'] self.logger.write( f'Successfully load pretrained model from {resume}!\n') self.logger.write(f'Current best loss {self.best_loss}\n') self.logger.write(f'Current best recall {self.best_recall}\n') else: raise ValueError(f"=> no checkpoint found at '{resume}'") def _get_lr(self, group=0): return self.optimizer.param_groups[group]['lr'] def stats_dict(self): stats = dict() stats['circle_loss'] = 0. stats[ 'recall'] = 0. # feature match recall, divided by number of ground truth pairs stats['saliency_loss'] = 0. stats['saliency_recall'] = 0. stats['saliency_precision'] = 0. stats['overlap_loss'] = 0. stats['overlap_recall'] = 0. stats['overlap_precision'] = 0. return stats def stats_meter(self): meters = dict() stats = self.stats_dict() for key, _ in stats.items(): meters[key] = AverageMeter() return meters def inference_one_batch(self, input_dict, phase): assert phase in ['train', 'val', 'test'] ################################## # training if (phase == 'train'): self.model.train() ############################################### # forward pass sinput_src = ME.SparseTensor(input_dict['src_F'].to(self.device), coordinates=input_dict['src_C'].to( self.device)) sinput_tgt = ME.SparseTensor(input_dict['tgt_F'].to(self.device), coordinates=input_dict['tgt_C'].to( self.device)) src_feats, tgt_feats, scores_overlap, scores_saliency = self.model( sinput_src, sinput_tgt) src_pcd, tgt_pcd = input_dict['pcd_src'].to( self.device), input_dict['pcd_tgt'].to(self.device) c_rot = input_dict['rot'].to(self.device) c_trans = input_dict['trans'].to(self.device) correspondence = input_dict['correspondences'].long().to( self.device) ################################################### # get loss stats = self.desc_loss(src_pcd, tgt_pcd, src_feats, tgt_feats, correspondence, c_rot, c_trans, scores_overlap, scores_saliency, input_dict['scale']) c_loss = stats['circle_loss'] * self.w_circle_loss + stats[ 'overlap_loss'] * self.w_overlap_loss + stats[ 'saliency_loss'] * self.w_saliency_loss c_loss.backward() else: self.model.eval() with torch.no_grad(): ############################################### # forward pass sinput_src = ME.SparseTensor( input_dict['src_F'].to(self.device), coordinates=input_dict['src_C'].to(self.device)) sinput_tgt = ME.SparseTensor( input_dict['tgt_F'].to(self.device), coordinates=input_dict['tgt_C'].to(self.device)) src_feats, tgt_feats, scores_overlap, scores_saliency = self.model( sinput_src, sinput_tgt) src_pcd, tgt_pcd = input_dict['pcd_src'].to( self.device), input_dict['pcd_tgt'].to(self.device) c_rot = input_dict['rot'].to(self.device) c_trans = input_dict['trans'].to(self.device) correspondence = input_dict['correspondences'].long().to( self.device) ################################################### # get loss stats = self.desc_loss(src_pcd, tgt_pcd, src_feats, tgt_feats, correspondence, c_rot, c_trans, scores_overlap, scores_saliency, input_dict['scale']) ################################## # detach the gradients for loss terms stats['circle_loss'] = float(stats['circle_loss'].detach()) stats['overlap_loss'] = float(stats['overlap_loss'].detach()) stats['saliency_loss'] = float(stats['saliency_loss'].detach()) return stats def inference_one_epoch(self, epoch, phase): gc.collect() assert phase in ['train', 'val', 'test'] # init stats meter stats_meter = self.stats_meter() num_iter = int( len(self.loader[phase].dataset) // self.loader[phase].batch_size) c_loader_iter = self.loader[phase].__iter__() self.optimizer.zero_grad() for c_iter in tqdm(range(num_iter)): # loop through this epoch inputs = c_loader_iter.next() try: ################################## # forward pass # with torch.autograd.detect_anomaly(): stats = self.inference_one_batch(inputs, phase) ################################################### # run optimisation if ((c_iter + 1) % self.iter_size == 0 and phase == 'train'): gradient_valid = validate_gradient(self.model) if (gradient_valid): self.optimizer.step() else: self.logger.write('gradient not valid\n') self.optimizer.zero_grad() ################################ # update to stats_meter for key, value in stats.items(): stats_meter[key].update(value) except RuntimeError as inst: pass torch.cuda.empty_cache() if (c_iter + 1) % self.verbose_freq == 0 and self.verbose: curr_iter = num_iter * (epoch - 1) + c_iter for key, value in stats_meter.items(): self.writer.add_scalar(f'{phase}/{key}', value.avg, curr_iter) message = f'{phase} Epoch: {epoch} [{c_iter+1:4d}/{num_iter}]' for key, value in stats_meter.items(): message += f'{key}: {value.avg:.2f}\t' self.logger.write(message + '\n') message = f'{phase} Epoch: {epoch}' for key, value in stats_meter.items(): message += f'{key}: {value.avg:.2f}\t' self.logger.write(message + '\n') return stats_meter def train(self): print('start training...') for epoch in range(self.start_epoch, self.max_epoch): self.inference_one_epoch(epoch, 'train') self.scheduler.step() stats_meter = self.inference_one_epoch(epoch, 'val') if stats_meter['circle_loss'].avg < self.best_loss: self.best_loss = stats_meter['circle_loss'].avg self._snapshot(epoch, 'best_loss') if stats_meter['recall'].avg > self.best_recall: self.best_recall = stats_meter['recall'].avg self._snapshot(epoch, 'best_recall') # we only add saliency loss when we get descent point-wise features if (stats_meter['recall'].avg > 0.3): self.w_saliency_loss = 1. else: self.w_saliency_loss = 0. # finish all epoch print("Training finish!") def eval(self): print('Start to evaluate on validation datasets...') stats_meter = self.inference_one_epoch(0, 'val') for key, value in stats_meter.items(): print(key, value.avg)
aux_dim, activation='lrelu', device=device, hidden_dim=args.hidden_dim, anneal=args.anneal) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=4, verbose=True) ste = time.time() print('setup time: {}s'.format(ste - st)) # setup loggers logger = Logger(path=LOG_FOLDER) exp_id = logger.get_id() tensorboard_run_name = TENSORBOARD_RUN_FOLDER + 'exp' + str( exp_id) + '_'.join( map(str, [ '', args.batch_size, args.max_iter, args.lr, args.hidden_dim, args.depth, args.anneal ])) writer = SummaryWriter(logdir=tensorboard_run_name) logger.add('elbo') logger.add('perf') print('Beginning training for exp: {}'.format(exp_id)) # training loop it = 0 model.train()
args = parser.parse_args() msg = 'Data file {} not found.' assert os.path.isfile(args.data_file), msg.format(args.data_file) msg = '{} directory not found. Please create it first.' assert os.path.isdir(args.data_dir), msg.format(args.data_dir) msg = 'the proportion of dataset to builded must in (0.0, 1.0]' assert (args.data_factor > 0.0) and (args.data_factor <= 1.0), msg msg = 'train factor + val factor + test factor must be equal to 1.0' total = args.train_factor + args.val_factor + args.test_factor assert (1.0 == total), msg dataset_cfg.set_data_dir(args.data_dir) # set and get logger logger = Logger.set(dataset_cfg.log_file()) # build, load and dump datasets builder = Builder(data_factor=args.data_factor, train_factor=args.train_factor, val_factor=args.val_factor, test_factor=args.test_factor, train_name=args.train_name, val_name=args.val_name, test_name=args.test_name, logger=logger) builder.load(args.data_file, encoding='windows-1252') builder.dump(dataset_cfg.data_dir(), dataset_cfg.params_file(), min_count_word=args.min_count_word, min_count_tag=args.min_count_tag,
# load parser dataset_cfg = DatasetCfg(config.data_dir) exp_cfg = ExperimentCfg(config.base_model_dir) parser = get_parser(data_dir=dataset_cfg.data_dir(), exp_dir=exp_cfg.experiment_dir(), restore_checkpoint=None) # parse command line arguments args = parser.parse_args() restore_checkpoint = args.restore_checkpoint dataset_cfg.set_data_dir(args.data_dir) exp_cfg.set_experiment_dir(args.exp_dir) # set logger # Note: log file will be stored in the `exp_dir` directory logger = Logger.set(exp_cfg.train_log()) # load experiment configuration logger.info("Loading the experiment configurations...") params = Params(exp_cfg.params_file()) logger.info("- done.") # set params params.set('cuda', torch.cuda.is_available()) # load datesets logger.info("Loading the datasets...") # add datasets parameters into params params.update(Params(dataset_cfg.params_file())) trainloader, valloader = load_data(params, dataset_cfg.data_dir(),
def __init__(self, name, config, topics, pages): self.name = name self.config = config self.topics = topics self.site_alias = { target['host']: target.get('alias', '') for target in self.config.targets } self.url_queries = Queue() self.current_info = Dict() self.queue_dict = { target['host']: Queue() for target in self.config.targets } self.current_info_dict = { target['host']: Dict() for target in self.config.targets } self.save_breakpoint_path = os.path.join(self.config.save.tmp, f'{self.name}.breakpoint') if os.path.isfile(self.save_breakpoint_path): self._load_breakpoint() self.request = Requestor(self.config) # self.searcher = Searcher(self.config) self.searcher = Searcher('baidu') self.parser = Parser() self.database = DataBase(**self.config.save.database.login) # self.reset_database() self.table = 'news' self.logger = Logger(filename=self.config.save.log) self.url_fetcher = SearchEngineUrlFetcher( request=self.request, searcher=self.searcher, queries=self.url_queries, current_info=self.current_info, hosts=[target['host'] for target in self.config.targets], topics=self.topics, pages=pages, queue_dict=self.queue_dict, ) # one thread for one host self.threads = [ TargetedCrawler( self.request, self.parser, self.url_fetcher, self.database, self.table, self.logger, target, self.queue_dict[target['host']], self.current_info_dict[target['host']], ) for target in config.targets ]
class TopicalCrawler: def __init__(self, name, config, topics, pages): self.name = name self.config = config self.topics = topics self.site_alias = { target['host']: target.get('alias', '') for target in self.config.targets } self.url_queries = Queue() self.current_info = Dict() self.queue_dict = { target['host']: Queue() for target in self.config.targets } self.current_info_dict = { target['host']: Dict() for target in self.config.targets } self.save_breakpoint_path = os.path.join(self.config.save.tmp, f'{self.name}.breakpoint') if os.path.isfile(self.save_breakpoint_path): self._load_breakpoint() self.request = Requestor(self.config) # self.searcher = Searcher(self.config) self.searcher = Searcher('baidu') self.parser = Parser() self.database = DataBase(**self.config.save.database.login) # self.reset_database() self.table = 'news' self.logger = Logger(filename=self.config.save.log) self.url_fetcher = SearchEngineUrlFetcher( request=self.request, searcher=self.searcher, queries=self.url_queries, current_info=self.current_info, hosts=[target['host'] for target in self.config.targets], topics=self.topics, pages=pages, queue_dict=self.queue_dict, ) # one thread for one host self.threads = [ TargetedCrawler( self.request, self.parser, self.url_fetcher, self.database, self.table, self.logger, target, self.queue_dict[target['host']], self.current_info_dict[target['host']], ) for target in config.targets ] def __call__(self, string, option='url'): if option == 'url': self.parse(string) elif option == 'keyword': urls = self.search(string) data = [] for url in urls: soup, url, host = self.request(url) data.append(self.parse(soup, host)) return data else: raise ValueError("option should be 'url' or 'keyword'") def test_search(self, keyword, host=None, page=1): query = self.searcher.construct_query(keyword, host, page) soup, _ = self.request(query) urls = self.searcher.parse(soup) return urls def test_parse(self, url, remain_query_key=None): soup, url = self.request(url, remain_query_key=remain_query_key, no_headers=True) data = self.parser(soup, url['netloc'], url['tld']) next_page_url = data.pop('next', None) while next_page_url: try: _soup, _url = self.request(next_page_url) _data = self.parser(_soup, _url['netloc'], _url['tld']) data['text'] += _data['text'] next_page_url = _data.pop('next', None) except: break return { **data, 'url': url['url'], 'site': self.site_alias.get(url['tld']) or self.site_alias.get(url['netloc'], '') } def _load_breakpoint(self): with open(self.save_breakpoint_path, 'rb') as f: temp = pickle.load(f) for q in temp['url_queries']: self.url_queries.put(q) for host, queue_ in temp['queue_dict'].items(): for q in queue_: self.queue_dict[host].put(q) def _save_breakpoint(self): temp = { 'url_queries': ([self.current_info.query] if self.current_info.get('query') else []) + [self.url_queries.get() for _ in range(self.url_queries.qsize())], 'queue_dict': { target: ([self.current_info_dict[target].query] if self.current_info_dict[target].get('query') else []) + [queue_.get() for _ in range(queue_.qsize())] for target, queue_ in self.queue_dict.items() } } with open(self.save_breakpoint_path, 'wb') as f: pickle.dump(temp, f) def _cleanup(self): try: self.database.close() except: pass def run(self): self._init_topics() self.url_fetcher.start() for t in self.threads: t.start() try: while True: time.sleep(1) if self.url_fetcher.is_alive(): continue for t in self.threads: if t.is_alive(): break else: break except KeyboardInterrupt: self.logger.info('keyboard interrupt by user') self._save_breakpoint() self._cleanup() except: self.logger.error(exc_info=sys.exc_info()) self._save_breakpoint() self._cleanup() def _init_topics(self): for topic in self.topics: if not isinstance(topic['keywords'], list): topic['keywords'] = [topic['keywords']] temp_keywords_str = json.dumps(topic['keywords']) if not self.database.select('topic', name=topic['name']): self.database.insert('topic', name=topic['name'], keywords=temp_keywords_str, entry_time=datetime.datetime.now(), remark=topic.get('remark', '')) else: self.database.update( { 'table': 'topic', 'constraints': { 'name': topic['name'] } }, remark=topic.get('remark', ''), keywords=temp_keywords_str)
'Handler': HandleBase, "Input": InputBase }) self.manager.collectPlugins() def _watiInputDone(self): plugins = self.manager.getPluginsOfCategory('Input') map(lambda plugin: plugin.plugin_object.proc.join(), plugins) def _inventorySync(self): zk_address = self.config.get("Inventory", "zk_address") full_sync(zk_address) def start(self): self._configInitialize() self._mangerInitialize() self._serviceInitialize() self._inventorySync() self._pluginInitialize() #self._watiInputDone() import time while True: time.sleep(30) if __name__ == "__main__": from lib.utils import Logger Logger.basicConfig() app = DmsOrchestrator() app.start()
def train_model(args, metadata, device='cuda'): print('training on {}'.format(torch.cuda.get_device_name(device) if args.cuda else 'cpu')) # load data if not args.preload: dset = SyntheticDataset(args.file, 'cpu') # originally 'cpu' ???? train_loader = DataLoader(dset, shuffle=True, batch_size=args.batch_size) data_dim, latent_dim, aux_dim = dset.get_dims() args.N = len(dset) metadata.update(dset.get_metadata()) else: train_loader = DataLoaderGPU(args.file, shuffle=True, batch_size=args.batch_size) data_dim, latent_dim, aux_dim = train_loader.get_dims() args.N = train_loader.dataset_len metadata.update(train_loader.get_metadata()) if args.max_iter is None: args.max_iter = len(train_loader) * args.epochs if args.latent_dim is not None: latent_dim = args.latent_dim metadata.update({"train_latent_dim": latent_dim}) # define model and optimizer model = None if args.i_what == 'iVAE': model = iVAE(latent_dim, data_dim, aux_dim, n_layers=args.depth, activation='lrelu', device=device, hidden_dim=args.hidden_dim, anneal=args.anneal, # False file=metadata['file'], # Added dataset location for easier checkpoint loading seed=1, epochs=args.epochs) elif args.i_what == 'iFlow': metadata.update({"device": device}) model = iFlow(args=metadata).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, \ factor=args.lr_drop_factor, \ patience=args.lr_patience, \ verbose=True) # factor=0.1 and patience=4 ste = time.time() print('setup time: {}s'.format(ste - st)) # setup loggers logger = Logger(logdir=LOG_FOLDER) # 'log/' exp_id = logger.get_id() # 1 tensorboard_run_name = TENSORBOARD_RUN_FOLDER + 'exp' + str(exp_id) + '_'.join( map(str, ['', args.batch_size, args.max_iter, args.lr, args.hidden_dim, args.depth, args.anneal])) # 'runs/exp1_64_12500_0.001_50_3_False' writer = SummaryWriter(logdir=tensorboard_run_name) if args.i_what == 'iFlow': logger.add('log_normalizer') logger.add('neg_log_det') logger.add('neg_trace') logger.add('loss') logger.add('perf') print('Beginning training for exp: {}'.format(exp_id)) # training loop epoch = 0 model.train() while epoch < args.epochs: # args.max_iter: #12500 est = time.time() for itr, (x, u, z) in enumerate(train_loader): acc_itr = itr + epoch * len(train_loader) # x is of shape [64, 4] # u is of shape [64, 40], one-hot coding of 40 classes # z is of shape [64, 2] # it += 1 # model.anneal(args.N, args.max_iter, it) optimizer.zero_grad() if args.cuda and not args.preload: x = x.cuda(device=device, non_blocking=True) u = u.cuda(device=device, non_blocking=True) if args.i_what == 'iVAE': elbo, z_est = model.elbo(x, u) # elbo is a scalar loss while z_est is of shape [64, 2] loss = elbo.mul(-1) elif args.i_what == 'iFlow': (log_normalizer, neg_trace, neg_log_det), z_est = model.neg_log_likelihood(x, u) loss = log_normalizer + neg_trace + neg_log_det loss.backward() optimizer.step() logger.update('loss', loss.item()) if args.i_what == 'iFlow': logger.update('log_normalizer', log_normalizer.item()) logger.update('neg_trace', neg_trace.item()) logger.update('neg_log_det', neg_log_det.item()) perf = mcc(z.cpu().numpy(), z_est.cpu().detach().numpy()) logger.update('perf', perf) if acc_itr % args.log_freq == 0: # % 25 logger.log() writer.add_scalar('data/performance', logger.get_last('perf'), acc_itr) writer.add_scalar('data/loss', logger.get_last('loss'), acc_itr) if args.i_what == 'iFlow': writer.add_scalar('data/log_normalizer', logger.get_last('log_normalizer'), acc_itr) writer.add_scalar('data/neg_trace', logger.get_last('neg_trace'), acc_itr) writer.add_scalar('data/neg_log_det', logger.get_last('neg_log_det'), acc_itr) scheduler.step(logger.get_last('loss')) if acc_itr % int(args.max_iter / 5) == 0 and not args.no_log: checkpoint(TORCH_CHECKPOINT_FOLDER, \ exp_id, \ acc_itr, \ model, \ optimizer, \ logger.get_last('loss'), \ logger.get_last('perf')) epoch += 1 eet = time.time() if args.i_what == 'iVAE': print('epoch {}: {:.4f}s;\tloss: {:.4f};\tperf: {:.4f}'.format(epoch, eet - est, logger.get_last('loss'), logger.get_last('perf'))) elif args.i_what == 'iFlow': print('epoch {}: {:.4f}s;\tloss: {:.4f} (l1: {:.4f}, l2: {:.4f}, l3: {:.4f});\tperf: {:.4f}'.format( \ epoch, eet - est, logger.get_last('loss'), logger.get_last('log_normalizer'), logger.get_last('neg_trace'), logger.get_last('neg_log_det'), logger.get_last('perf'))) et = time.time() print('training time: {}s'.format(et - ste)) # Save final model checkpoint(PT_MODELS_FOLDER, "", 'final', model, optimizer, logger.get_last('loss'), logger.get_last('perf')) writer.close() if not args.no_log: logger.add_metadata(**metadata) logger.save_to_json() logger.save_to_npz() print('total time: {}s'.format(et - st)) return model
prediction, to_device(ones_target(size), device) ) # instead of minimizing log(1-D(G(z))), maximise log(D(gz)) for stronger gradients in early training error.backward() ''' Update weights with gradients ''' optimizer.step() return error # Testing num_test_samples = 3 test_noise = to_device(noise(num_test_samples), device) # Create logger instance logger = Logger(model_name='GAN_d' + str(d_lr) + '_g' + str(g_lr), data_name='EPianoDataset') #C:\Users\Dave\PycharmProjects\DataLoading\runs # Printing model & optimizer state_dict print("Discriminator state_dict:") for param_tensor in discriminator.state_dict(): print(param_tensor, "\t", discriminator.state_dict()[param_tensor].size()) print("\nGenerator state_dict:") for param_tensor in generator.state_dict(): print(param_tensor, "\t", generator.state_dict()[param_tensor].size()) print("\nD_Optimizer state_dict:") for var_name in d_optimizer.state_dict(): print(var_name, "\t", d_optimizer.state_dict()[var_name])
from lib.trainer import Trainer from lib.utils import load_model,save_model,Logger from lib.coco import COCO from lib import optimer from config import cfg as opt import torch import os torch.backends.cudnn.benchmark= True ## input size is not fixed os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.gpus = [int(i) for i in opt.gpus_str.split(',')] opt.gpus = list(range(len(opt.gpus))) opt.batch_size = opt.batch_size * len(opt.gpus) opt.save_dir = os.path.join(opt.save_dir,opt.exp_id) logger = Logger(opt) model = TensorMask(backbone=opt.backbone , num_cls=opt.num_class , base_window= opt.base_window , freezeBN=opt.frezeBN,freezeLayers=opt.frezeLayer, align_corners= opt.align_corners) optimizer = optimer.SGD([{'params':filter(lambda x:len(x.size()) == 4 ,model.parameters()),'weight_decay':0.0001 }, {'params': filter(lambda x:len(x.size()) <4,model.parameters())}], lr=opt.lr,warm_up=1000,momentum=0.9,nesterov=True) start_epoch = 0 if opt.weights != '' : model, optimizer, start_epoch = load_model( model, opt.weights, optimizer, opt.resume, opt.lr, opt.lr_step) trainer = Trainer(opt,model,optimizer)
shutil.copyfile(filename, best_file) print('=> testing accuracy of last model') test_all(model, base_loader, base_val_loader, val_loader, test_loader) if os.path.isfile(os.path.join(cfg.misc.checkpoint_dir, 'best_model.tar')): # release GPU memory used by benchmark to avoid OOM torch.cuda.empty_cache() model, _, resume_epoch, best_acc, _ = load_checkpoint( model, optimizer, 'best') print( f'=> testing accuracy of best model in {resume_epoch} epoch with best validate accuracy {best_acc}' ) test_all(model, base_loader, base_val_loader, val_loader, test_loader) if __name__ == '__main__': sys.stdout = Logger(os.path.join(cfg.misc.output_dir, 'log.txt')) print('======CONFIGURATION START======') pprint(cfg) print('======CONFIGURATION END======') # for reproducibility np.random.seed(cfg.misc.rng_seed) torch.manual_seed(cfg.misc.rng_seed) torch.backends.cudnn.deterministic = True # for efficient torch.backends.cudnn.benchmark = True main()
def main(): global args set_random_seed(args.seed) if not args.use_avai_gpus: os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_devices use_gpu = torch.cuda.is_available() and not args.use_cpu log_name = 'test.log' if args.evaluate else 'train.log' log_name += time.strftime('-%Y-%m-%d-%H-%M-%S') sys.stdout = Logger(osp.join(args.save_dir, log_name)) print('** Arguments **') arg_keys = list(args.__dict__.keys()) arg_keys.sort() for key in arg_keys: print('{}: {}'.format(key, args.__dict__[key])) print('\n') print('Collecting env info ...') print('** System info **\n{}\n'.format(collect_env_info())) if use_gpu: torch.backends.cudnn.benchmark = True # torch.backends.cudnn.benchmark = False # torch.backends.cudnn.deterministic = True else: warnings.warn('Currently using CPU, however, GPU is highly recommended') # load data related args data_args = imagedata_kwargs(args) # initialize dataset dataset = init_image_dataset(name=data_args['source'], **data_args) # build data transformer transforms_tr, transforms_te = build_transforms(**data_args) # load train data trainset = dataset.train train_sampler = build_train_sampler( trainset, data_args['train_sampler'], batch_size=data_args['batch_size'], num_instances=data_args['num_instances'], num_train_pids=dataset.num_train_pids ) trainloader = torch.utils.data.DataLoader( DataWarpper(data=trainset, transforms=transforms_tr), sampler=train_sampler, batch_size=data_args['batch_size'], shuffle=False, num_workers=data_args['workers'], pin_memory=False, drop_last=True, ) # load test data queryset = dataset.query queryloader = torch.utils.data.DataLoader( DataWarpper(data=queryset, transforms=transforms_te), batch_size=data_args['batch_size'], shuffle=False, num_workers=data_args['workers'], pin_memory=False, drop_last=False ) galleryset = dataset.gallery galleryloader = torch.utils.data.DataLoader( DataWarpper(data=galleryset, transforms=transforms_te), batch_size=data_args['batch_size'], shuffle=False, num_workers=data_args['workers'], pin_memory=False, drop_last=False ) print('Building model: {}'.format(args.arch)) model = build_model( name=args.arch, num_classes=dataset.num_train_pids, pretrained=(not args.no_pretrained), use_gpu=use_gpu, batch_size=args.batch_size, part_num=args.part_num, part_weight=args.part_weight ) model = model.cuda() # num_params, flops = compute_model_complexity(model, (1, 3, args.height, args.width)) # print('Model complexity: params={:,} flops={:,}'.format(num_params, flops)) if args.load_weights and check_isfile(args.load_weights): load_pretrained_weights(model, args.load_weights) optimizer = build_optimizer(model, **optimizer_kwargs(args)) scheduler = build_lr_scheduler(optimizer, **lr_scheduler_kwargs(args)) model, optimizer = amp.initialize(model, optimizer, opt_level="O1", keep_batchnorm_fp32=None, loss_scale=None) if use_gpu: model = nn.DataParallel(model) if args.resume and check_isfile(args.resume): args.start_epoch = resume_from_checkpoint(args.resume, model, optimizer=optimizer) print('Building {}-engine for {}-reid'.format(args.loss, args.app)) engine = Engine(trainloader, queryloader, galleryloader, model, optimizer, scheduler, query=queryset, gallery=galleryset, use_gpu=use_gpu, num_train_pids=dataset.num_train_pids, **engine_kwargs(args)) engine.run(**engine_kwargs(args), use_gpu=use_gpu)
if args.split_pos: plt.subplots(figsize=(30, 16)) plot(2, 1, y_data[:, :args.split_pos], y_data_std[:, :args.split_pos], args.initial_x_1, args.title_1, args.x_label, args.y_label) plot(2, 2, y_data[:, args.split_pos:], y_data_std[:, args.split_pos:], args.initial_x_2, args.title_2, args.x_label, args.y_label) else: plt.subplots(figsize=(30, 3)) plot(1, 1, y_data, y_data_std, args.initial_x, args.title, args.x_label, args.y_label) plt.savefig("%s.png" % args.prefix, dpi=300, bbox_inches='tight') plt.close() log = Logger() if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--reference", help="The reference network (.dat)") parser.add_argument( "--reference-std", help= "The reference standard deviation network (.dat) - should be in identical order as alternative networks" ) parser.add_argument("--alternatives", help="The alternative networks (.dat)", nargs="*") parser.add_argument( "--alternatives-std",
N_HIDDEN = 388 N_OUTPUTS = 388 N_EPOCHS = 10 trainset = DeviceDataLoader( DataLoader(EpianoDataset(N_STEPS), BATCH_SIZE, shuffle=True), device) num_batches = len(trainset) # number of batches model = EPianoLSTM(BATCH_SIZE, N_LAYERS, N_STEPS, N_INPUTS, N_HIDDEN, N_OUTPUTS) to_device(model, device) optimizer = optim.Adam(model.parameters(), lr=0.001) lossF = nn.MSELoss() # Create logger instance logger = Logger(model_name='LSTM', data_name='EPianoDataset') print("LSTM state_dict:") for param_tensor in model.state_dict(): print(param_tensor, "\t", model.state_dict()[param_tensor].size()) print("\nOptimizer state_dict:") for var_name in optimizer.state_dict(): print(var_name, "\t", optimizer.state_dict()[var_name]) """ TRAINING """ for epoch in range(N_EPOCHS): train_running_loss = 0.0 train_accuracy = 0.0 iterations = 0
return scheduler def _serviceInitialize(self): ctx = ServiceContext() queue = Queue() ctx.registerQueueService(queue) ctx.registerSchedService(self.initializeScheduler()) def _pluginInitialize(self): self.manager.setPluginPlaces(["lib/plugins"]) self.manager.setConfigParser(self.config,None) self.manager.setCategoriesFilter({'Handler':HandleBase,"Input":InputBase}) self.manager.collectPlugins() def _watiInputDone(self): plugins = self.manager.getPluginsOfCategory('Input') map(lambda plugin:plugin.plugin_object.proc.join(),plugins) def start(self): self._configInitialize() self._mangerInitialize() self._serviceInitialize() self._pluginInitialize() self._watiInputDone() if __name__ == "__main__": from lib.utils import Logger Logger.basicConfig() app = DmsOrchestrator() app.start()
def main(model, auxiliary=True, model_label='rcnn', rnn_type='gru', padding='pre', reg='s', prefix="crawl", embedding_file_type="word2vec", train_fname="./data/train.csv", test_fname="./data/test.csv", embeds_fname="./data/GoogleNews-vectors-negative300.bin", logger_fname="./logs/log-aws", mode="all", wrong_words_fname="./data/correct_words.csv", format_embeds="binary", config="./config.json", output_dir="./out", norm_prob=False, norm_prob_koef=1, gpus=0, char_level=False, random_seed=2018, num_folds=5): embedding_type = prefix + "_" + embedding_file_type logger = Logger(logging.getLogger(), logger_fname) # ====Detect GPUs==== logger.debug(device_lib.list_local_devices()) # ====Load data==== logger.info('Loading data...') train_df = load_data(train_fname) test_df = load_data(test_fname) target_labels = [ 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate' ] num_classes = len(target_labels) # ====Load additional data==== logger.info('Loading additional data...') # swear_words = load_data(swear_words_fname, func=lambda x: set(x.T[0]), header=None) wrong_words_dict = load_data(wrong_words_fname, func=lambda x: {val[0]: val[1] for val in x}) tokinizer = RegexpTokenizer(r'\S+') regexps = [ re.compile("([a-zA-Z]+)([0-9]+)"), re.compile("([0-9]+)([a-zA-Z]+)") ] # ====Load word vectors==== logger.info('Loading embeddings...') if model != 'mvcnn': embed_dim = 300 embeds = Embeds(embeds_fname, embedding_file_type, format=format_embeds) if mode in ('preprocess', 'all'): logger.info('Generating indirect features...') # https://www.kaggle.com/jagangupta/stop-the-s-toxic-comments-eda # Word count in each comment: train_df['count_word'] = train_df["comment_text"].apply( lambda x: len(str(x).split())) test_df['count_word'] = test_df["comment_text"].apply( lambda x: len(str(x).split())) # Unique word count train_df['count_unique_word'] = train_df["comment_text"].apply( lambda x: len(set(str(x).split()))) test_df['count_unique_word'] = test_df["comment_text"].apply( lambda x: len(set(str(x).split()))) # Letter count train_df['count_letters'] = train_df["comment_text"].apply( lambda x: len(str(x))) test_df['count_letters'] = test_df["comment_text"].apply( lambda x: len(str(x))) # punctuation count train_df["count_punctuations"] = train_df["comment_text"].apply( lambda x: len([c for c in str(x) if c in string.punctuation])) test_df["count_punctuations"] = test_df["comment_text"].apply( lambda x: len([c for c in str(x) if c in string.punctuation])) # upper case words count train_df["count_words_upper"] = train_df["comment_text"].apply( lambda x: len([w for w in str(x).split() if w.isupper()])) test_df["count_words_upper"] = test_df["comment_text"].apply( lambda x: len([w for w in str(x).split() if w.isupper()])) # title case words count train_df["count_words_title"] = train_df["comment_text"].apply( lambda x: len([w for w in str(x).split() if w.istitle()])) test_df["count_words_title"] = test_df["comment_text"].apply( lambda x: len([w for w in str(x).split() if w.istitle()])) # Word count percent in each comment: train_df['word_unique_pct'] = train_df[ 'count_unique_word'] * 100 / train_df['count_word'] test_df['word_unique_pct'] = test_df[ 'count_unique_word'] * 100 / test_df['count_word'] # Punct percent in each comment: train_df['punct_pct'] = train_df[ 'count_punctuations'] * 100 / train_df['count_word'] test_df['punct_pct'] = test_df['count_punctuations'] * 100 / test_df[ 'count_word'] # Average length of the words train_df["mean_word_len"] = train_df["comment_text"].apply( lambda x: np.mean([len(w) for w in str(x).split()])) test_df["mean_word_len"] = test_df["comment_text"].apply( lambda x: np.mean([len(w) for w in str(x).split()])) # upper case words percentage train_df["words_upper_pct"] = train_df[ "count_words_upper"] * 100 / train_df['count_word'] test_df["words_upper_pct"] = test_df[ "count_words_upper"] * 100 / test_df['count_word'] # title case words count train_df["words_title_pct"] = train_df[ "count_words_title"] * 100 / train_df['count_word'] test_df["words_title_pct"] = test_df[ "count_words_title"] * 100 / test_df['count_word'] # remove columns train_df = train_df.drop('count_word', 1) train_df = train_df.drop('count_unique_word', 1) train_df = train_df.drop('count_punctuations', 1) train_df = train_df.drop('count_words_upper', 1) train_df = train_df.drop('count_words_title', 1) test_df = test_df.drop('count_word', 1) test_df = test_df.drop('count_unique_word', 1) test_df = test_df.drop('count_punctuations', 1) test_df = test_df.drop('count_words_upper', 1) test_df = test_df.drop('count_words_title', 1) logger.info('Cleaning text...') train_df['comment_text_clear'] = clean_text(train_df['comment_text'], tokinizer, wrong_words_dict, regexps, autocorrect=False) test_df['comment_text_clear'] = clean_text(test_df['comment_text'], tokinizer, wrong_words_dict, regexps, autocorrect=False) if reg == 'w': # remove all punctuations train_df.to_csv(os.path.join(output_dir, 'train_clear_w.csv'), index=False) test_df.to_csv(os.path.join(output_dir, 'test_clear_w.csv'), index=False) train_df = pd.read_csv( os.path.join(output_dir, 'train_clear_w.csv')) test_df = pd.read_csv(os.path.join(output_dir, 'test_clear_w.csv')) elif reg == 's': # split by S+ keep all punctuations train_df.to_csv(os.path.join(output_dir, 'train_clear.csv'), index=False) test_df.to_csv(os.path.join(output_dir, 'test_clear.csv'), index=False) train_df = pd.read_csv(os.path.join(output_dir, 'train_clear.csv')) test_df = pd.read_csv(os.path.join(output_dir, 'test_clear.csv')) if mode == 'preprocess': return if mode == 'processed': if reg == 'w': train_df = pd.read_csv( os.path.join(output_dir, 'train_clear_w.csv')) test_df = pd.read_csv(os.path.join(output_dir, 'test_clear_w.csv')) elif reg == 's': train_df = pd.read_csv(os.path.join(output_dir, 'train_clear.csv')) test_df = pd.read_csv(os.path.join(output_dir, 'test_clear.csv')) logger.info('Calc text length...') train_df.fillna('unknown', inplace=True) test_df.fillna('unknown', inplace=True) train_df['text_len'] = train_df['comment_text_clear'].apply( lambda words: len(words.split())) test_df['text_len'] = test_df['comment_text_clear'].apply( lambda words: len(words.split())) max_seq_len = np.round(train_df['text_len'].mean() + 3 * train_df['text_len'].std()).astype(int) logger.debug('Max seq length = {}'.format(max_seq_len)) # ====Prepare data to NN==== logger.info('Converting texts to sequences...') max_words = 100000 if char_level: max_seq_len = 1200 train_df['comment_seq'], test_df[ 'comment_seq'], word_index = convert_text2seq( train_df['comment_text_clear'].tolist(), test_df['comment_text_clear'].tolist(), max_words, max_seq_len, embeds, lower=True, char_level=char_level, uniq=True, use_only_exists_words=True, position=padding) logger.debug('Dictionary size = {}'.format(len(word_index))) logger.info('Preparing embedding matrix...') if model != 'mvcnn': embedding_matrix, words_not_found = get_embedding_matrix( embed_dim, embeds, max_words, word_index) logger.debug('Embedding matrix shape = {}'.format( np.shape(embedding_matrix))) logger.debug('Number of null word embeddings = {}'.format( np.sum(np.sum(embedding_matrix, axis=1) == 0))) # ====Train/test split data==== # train/val x_aux = np.matrix([ train_df["word_unique_pct"].tolist(), train_df["punct_pct"].tolist(), train_df["mean_word_len"].tolist(), train_df["words_upper_pct"].tolist(), train_df["words_title_pct"].tolist() ], dtype='float32').transpose((1, 0)) x = np.array(train_df['comment_seq'].tolist()) y = np.array(train_df[target_labels].values) x_train_nn, x_test_nn, x_aux_train_nn, x_aux_test_nn, y_train_nn, y_test_nn, train_idxs, test_idxs = \ split_data(x, np.squeeze(np.asarray(x_aux)),y,test_size=0.2,shuffle=True,random_state=2018) # test set test_df_seq = np.array(test_df['comment_seq'].tolist()) test_aux = np.matrix([ train_df["word_unique_pct"].tolist(), train_df["punct_pct"].tolist(), train_df["mean_word_len"].tolist(), train_df["words_upper_pct"].tolist(), train_df["words_title_pct"].tolist() ], dtype='float32').transpose((1, 0)) test_df_seq_aux = np.squeeze(np.asarray(test_aux)) y_nn = [] logger.debug('X shape = {}'.format(np.shape(x_train_nn))) # ====Train models==== params = Params(config) if model_label == None: logger.warn('Should choose a model to train') return if model_label == 'dense': model = dense( embedding_matrix, num_classes, max_seq_len, dense_dim=params.get('dense').get('dense_dim'), n_layers=params.get('dense').get('n_layers'), concat=params.get('dense').get('concat'), dropout_val=params.get('dense').get('dropout_val'), l2_weight_decay=params.get('dense').get('l2_weight_decay'), pool=params.get('dense').get('pool'), train_embeds=params.get('dense').get('train_embeds'), add_sigmoid=True, gpus=gpus) if model_label == 'cnn': model = cnn(embedding_matrix, num_classes, max_seq_len, num_filters=params.get('cnn').get('num_filters'), l2_weight_decay=params.get('cnn').get('l2_weight_decay'), dropout_val=params.get('cnn').get('dropout_val'), dense_dim=params.get('cnn').get('dense_dim'), train_embeds=params.get('cnn').get('train_embeds'), n_cnn_layers=params.get('cnn').get('n_cnn_layers'), pool=params.get('cnn').get('pool'), add_embeds=params.get('cnn').get('add_embeds'), auxiliary=auxiliary, add_sigmoid=True, gpus=gpus) if model_label == 'cnn2d': model = cnn2d( embedding_matrix, num_classes, max_seq_len, num_filters=params.get('cnn2d').get('num_filters'), l2_weight_decay=params.get('cnn2d').get('l2_weight_decay'), dropout_val=params.get('cnn2d').get('dropout_val'), dense_dim=params.get('cnn2d').get('dense_dim'), train_embeds=params.get('cnn2d').get('train_embeds'), add_embeds=params.get('cnn2d').get('add_embeds'), auxiliary=auxiliary, add_sigmoid=True, gpus=gpus) if model_label == 'lstm': model = rnn( embedding_matrix, num_classes, max_seq_len, l2_weight_decay=params.get('lstm').get('l2_weight_decay'), rnn_dim=params.get('lstm').get('rnn_dim'), dropout_val=params.get('lstm').get('dropout_val'), dense_dim=params.get('lstm').get('dense_dim'), n_branches=params.get('lstm').get('n_branches'), n_rnn_layers=params.get('lstm').get('n_rnn_layers'), n_dense_layers=params.get('lstm').get('n_dense_layers'), train_embeds=params.get('lstm').get('train_embeds'), mask_zero=params.get('lstm').get('mask_zero'), kernel_regularizer=params.get('lstm').get('kernel_regularizer'), recurrent_regularizer=params.get('lstm').get( 'recurrent_regularizer'), activity_regularizer=params.get('lstm').get( 'activity_regularizer'), dropout=params.get('lstm').get('dropout'), recurrent_dropout=params.get('lstm').get('recurrent_dropout'), auxiliary=auxiliary, add_sigmoid=True, gpus=gpus, rnn_type='lstm') if model_label == 'gru': model = rnn( embedding_matrix, num_classes, max_seq_len, l2_weight_decay=params.get('gru').get('l2_weight_decay'), rnn_dim=params.get('gru').get('rnn_dim'), dropout_val=params.get('gru').get('dropout_val'), dense_dim=params.get('gru').get('dense_dim'), n_branches=params.get('gru').get('n_branches'), n_rnn_layers=params.get('gru').get('n_rnn_layers'), n_dense_layers=params.get('gru').get('n_dense_layers'), train_embeds=params.get('gru').get('train_embeds'), mask_zero=params.get('gru').get('mask_zero'), kernel_regularizer=params.get('gru').get('kernel_regularizer'), recurrent_regularizer=params.get('gru').get( 'recurrent_regularizer'), activity_regularizer=params.get('gru').get('activity_regularizer'), dropout=params.get('gru').get('dropout'), recurrent_dropout=params.get('gru').get('recurrent_dropout'), auxiliary=auxiliary, add_sigmoid=True, gpus=gpus, rnn_type='gru') if model_label == 'charrnn': model = charrnn( len(word_index), num_classes, max_seq_len, rnn_dim=params.get('charrnn').get('rnn_dim'), dropout_val=params.get('charrnn').get('dropout_val'), auxiliary=auxiliary, dropout=params.get('charrnn').get('dropout'), recurrent_dropout=params.get('charrnn').get('recurrent_dropout'), add_sigmoid=True, gpus=gpus, rnn_type=rnn_type) if model_label == 'cnn2rnn': model = cnn2rnn(embedding_matrix, num_classes, max_seq_len, rnn_type=rnn_type) if model_label == 'dpcnn': model = dpcnn(embedding_matrix, num_classes, max_seq_len, num_filters=params.get('dpcnn').get('num_filters'), dense_dim=params.get('dpcnn').get('dense_dim'), add_sigmoid=True, gpus=gpus) if model_label == 'rcnn': model = rcnn( embedding_matrix, num_classes, max_seq_len, rnn_dim=params.get('rcnn').get('rnn_dim'), dropout_val=params.get('rcnn').get('dropout_val'), dense_dim=params.get('rcnn').get('dense_dim'), train_embeds=params.get('rcnn').get('train_embeds'), auxiliary=auxiliary, dropout=params.get('rcnn').get('dropout'), recurrent_dropout=params.get('rcnn').get('recurrent_dropout'), add_sigmoid=True, gpus=gpus, rnn_type=rnn_type) if model_label == 'capsule': model = capsule( embedding_matrix, num_classes, max_seq_len, auxiliary=auxiliary, Num_capsule=params.get('capsule').get('Num_capsule'), Routings=params.get('capsule').get('Routing'), add_sigmoid=params.get('capsule').get('add_sigmoid'), mask_zero=params.get('capsule').get('mask_zero'), gpus=gpus, rnn_type='gru') # lstm may diverge but gru works better if model == 'mvcnn': embeds_fname1 = "./data/crawl-300d-2M.vec" # "./data/crawl-300d-2M.vec word2vec-raw.txt embeds_fname2 = "./data/glove.840B.300d.txt" embeds_fname3 = "./data/GoogleNews-vectors-negative300.bin" embed_dim = 300 embeds1 = Embeds(embeds_fname1, "glove", format='file') embeds2 = Embeds(embeds_fname2, "fasttext", format='file') embeds3 = Embeds(embeds_fname3, "word2vec", format='binary') embedding_matrix1, words_not_found1 = get_embedding_matrix( embed_dim, embeds1, max_words, word_index) embedding_matrix2, words_not_found2 = get_embedding_matrix( embed_dim, embeds2, max_words, word_index) #embedding_matrix3, words_not_found3 = get_embedding_matrix(embed_dim, embeds3, max_words, word_index) model = mvcnn(embedding_matrix1, embedding_matrix2, num_classes, max_seq_len, auxiliary=auxiliary, gpus=gpus) # ====k-fold cross validations split data==== logger.info('Run k-fold cross validation...') params = Params(config) kf = KFold(n_splits=num_folds, shuffle=True, random_state=random_seed) oof_train = np.zeros((x.shape[0], num_classes)) oof_test_skf = [] for i, (train_index, test_index) in enumerate(kf.split(x, y)): print("TRAIN:", train_index, "TEST:", test_index) x_train, x_aux_train, x_test, x_aux_test = x[train_index], x_aux[ train_index], x[test_index], x_aux[test_index] y_train, y_test = y[train_index], y[test_index] logger.info('Start training {}-th fold'.format(i)) if auxiliary: inputs = [x_train, x_aux_train] inputs_val = [x_test, x_aux_test] output = [test_df_seq, test_df_seq_aux] else: inputs = x_train inputs_val = x_test output = test_df_seq hist = train( x_train= inputs, # [x_train, x_aux_train] when auxiliary input is allowed. y_train=y_train, x_val=inputs_val, # [x_test, x_aux_test], y_val=y_test, model=model, batch_size=params.get(model_label).get('batch_size'), num_epochs=params.get(model_label).get('num_epochs'), learning_rate=params.get(model_label).get('learning_rate'), early_stopping_delta=params.get(model_label).get( 'early_stopping_delta'), early_stopping_epochs=params.get(model_label).get( 'early_stopping_epochs'), use_lr_strategy=params.get(model_label).get('use_lr_strategy'), lr_drop_koef=params.get(model_label).get('lr_drop_koef'), epochs_to_drop=params.get(model_label).get('epochs_to_drop'), model_checkpoint_dir=os.path.join('.', 'model_checkpoint', reg, model_label, embedding_type, padding, str(i)), logger=logger) model.load_weights( os.path.join('.', 'model_checkpoint', reg, model_label, embedding_type, padding, str(i), 'weights.h5')) oof_train[test_index, :] = model.predict( inputs_val) # model.predict([x_test, x_aux_test]) proba = model.predict( output) # model.predict([test_df_seq, test_df_seq_aux]) oof_test_skf.append(proba) result = pd.read_csv("./data/sample_submission.csv") result[target_labels] = proba ithfold_path = "./cv/{}/{}/{}/{}/{}".format(reg, model_label, embedding_type, padding, i) if not os.path.exists(ithfold_path): os.makedirs(ithfold_path) result.to_csv(os.path.join(ithfold_path, 'sub.csv'), index=False) # model.save(os.path.join(ithfold_path,'weights.h5')) # dump oof_test and oof_train for later slacking # oof_train: oof_train_path = "./cv/{}/{}/{}/{}/oof_train".format( reg, model_label, embedding_type, padding) if not os.path.exists(oof_train_path): os.makedirs(oof_train_path) np.savetxt(os.path.join(oof_train_path, "oof_train.csv"), oof_train, fmt='%.24f', delimiter=' ') # oof_test: stacking version oof_test = np.array(oof_test_skf).mean(axis=0) oof_test_path = "./cv/{}/{}/{}/{}/oof_test".format(reg, model_label, embedding_type, padding) if not os.path.exists(oof_test_path): os.makedirs(oof_test_path) np.savetxt(os.path.join(oof_test_path, "oof_test.csv"), oof_test, fmt='%.24f', delimiter=' ') # oof_test: submission version result[target_labels] = oof_test oof_test_bag_path = "./cv/{}/{}/{}/{}/bagged".format( reg, model_label, embedding_type, padding) if not os.path.exists(oof_test_bag_path): os.makedirs(oof_test_bag_path) result.to_csv(os.path.join(oof_test_bag_path, "sub.csv"), index=False)
from multiprocessing import Pool import align.detect_face as detect_face import cv2 import numpy as np # import tensorflow as tf import tensorflow.compat.v1 as tf tf.disable_v2_behavior() from lib.face_utils import judge_side_face from lib.utils import Logger, mkdir from project_root_dir import project_dir from src.sort import Sort logger = Logger() def main(): opencv() def testMultiProcess(): p = Pool(processes=2) sonuclar = p.map(detectExtract, ["1", "2"]) def opencv(): video_name = os.path.join("videos", "istiklal2.mp4") cam = cv2.VideoCapture(video_name) frames = []
def train(word_emb, vision_model, language_model, ent_loss_model, rel_loss_model, train_loader, val_loader, word_dict, ent_dict, pred_dict, n_epochs, val_freq, out_dir, cfg, grad_freq=0): os.makedirs(out_dir, exist_ok=True) params = list(vision_model.parameters()) + list( language_model.parameters()) params = [param for param in params if param.requires_grad] named_params = list(vision_model.named_parameters()) + list( language_model.named_parameters()) optimizer = torch.optim.Adam(params, lr=cfg.train.learning_rate, weight_decay=cfg.train.weight_decay) scheduler = torch.optim.lr_scheduler.StepLR( optimizer, step_size=1, gamma=cfg.train.learning_rate_decay) logger = Logger(os.path.join(out_dir, "log.txt")) tfb_logger = TFBLogger(out_dir) if grad_freq > 0: plt.ion() n_batches = len(train_loader) step = 0 for epoch in range(n_epochs): scheduler.step() epoch_loss = 0.0 if epoch % val_freq == 0: vision_model.train(False) language_model.train(False) ent_acc, rel_acc = validate(word_emb, vision_model, language_model, val_loader, word_dict, ent_dict, pred_dict, cfg.language_model.tokens_length, tfb_logger, step) logstr = "epoch %2d | ent acc(top20): %.3f | rel acc(top20): %.3f" % ( epoch, ent_acc, rel_acc) logger.write("%-80s" % logstr) vision_model.train(True) language_model.train(True) tic_0 = time.time() for i, data in enumerate(train_loader): tic_1 = time.time() image_ids = data[0] if len(image_ids) < cfg.train.batch_size: continue images = data[1].cuda().float() sbj_boxes = data[2].cuda().float() obj_boxes = data[3].cuda().float() rel_boxes = data[4].cuda().float() sbj_tokens = data[5].cuda() obj_tokens = data[6].cuda() rel_tokens = data[7].cuda() sbj_seq_lens = data[8].cuda().long() obj_seq_lens = data[9].cuda().long() rel_seq_lens = data[10].cuda().long() tic_2 = time.time() optimizer.zero_grad() sbj_t_emb = language_model(word_emb(sbj_tokens), sbj_seq_lens) obj_t_emb = language_model(word_emb(obj_tokens), obj_seq_lens) rel_t_emb = language_model(word_emb(rel_tokens), rel_seq_lens) sbj_v_emb, obj_v_emb, rel_v_emb = vision_model( images, sbj_boxes, obj_boxes, rel_boxes) sbj_loss = ent_loss_model(sbj_v_emb, sbj_t_emb) obj_loss = ent_loss_model(obj_v_emb, obj_t_emb) rel_loss = rel_loss_model(rel_v_emb, rel_t_emb) loss = sbj_loss + obj_loss + rel_loss tic_3 = time.time() loss.backward() optimizer.step() tic_4 = time.time() if grad_freq > 0 and i % grad_freq == 0: for n, p in named_params: if not "bias" in n: name_path = n.replace(".", "/") tfb_logger.histo_summary("grad/%s" % name_path, p.grad.data.cpu().numpy(), step) epoch_loss += loss.item() * train_loader.batch_size logstr = "epoch %2d batch %4d/%4d | loss %5.2f | %4dms | ^ %4dms | => %4dms" % \ (epoch+1, i+1, n_batches, loss.item(), 1000*(tic_4-tic_0), 1000*(tic_2-tic_0), 1000*(tic_4-tic_2)) print("%-80s" % logstr, end="\r") tfb_logger.scalar_summary("loss/ent", sbj_loss.item() + obj_loss.item(), step) tfb_logger.scalar_summary("loss/rel", rel_loss.item(), step) tfb_logger.scalar_summary("loss/total", loss.item(), step) tic_0 = time.time() step += train_loader.batch_size epoch_loss /= n_batches * train_loader.batch_size logstr = "epoch %2d | train_loss: %.3f" % (epoch + 1, epoch_loss) logger.write("%-80s" % logstr) vision_model_path = os.path.join(out_dir, "vision_model_%d.pth" % (epoch + 1)) torch.save(vision_model.state_dict(), vision_model_path) language_model_path = os.path.join( out_dir, "language_model_%d.pth" % (epoch + 1)) torch.save(language_model.state_dict(), language_model_path)