np.random.seed(args.rand_seed) if args.verbose >= 1: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s') for arg in vars(args): print("{}={}".format(arg, getattr(args, arg))) # get vocabulary logging.info('Extracting words from ' + args.train_set) vocab = dh.get_vocabulary(args.train_set, include_caption=args.include_caption) # load data logging.info('Loading training data from ' + args.train_set) train_data = dh.load(args.fea_type, args.train_path, args.train_set, include_caption=args.include_caption, separate_caption=args.separate_caption, vocab=vocab, max_history_length=args.max_history_length, merge_source=args.merge_source) logging.info('Loading validation data from ' + args.valid_set) valid_data = dh.load(args.fea_type, args.valid_path, args.valid_set, include_caption=args.include_caption, separate_caption=args.separate_caption, vocab=vocab, max_history_length=args.max_history_length, merge_source=args.merge_source) if args.fea_type[0] == 'none': feature_dims = 0 else: feature_dims = dh.feature_shape(train_data) logging.info("Detected feature dims: {}".format(feature_dims)); # report data summary logging.info('#vocab = %d' % len(vocab)) # make batchset for training train_indices, train_samples = dh.make_batch_indices(train_data, args.batch_size,
# load data vocab = {'<unk>': 0, '<sos>': 1, '<eos>': 2, '<no_tag>': 3} data = [] enc_hsizes = [] enc_psizes = [] for a in range(0, 1): feafile = args.feafile print('Loading data from', feafile, 'and', args.capfile) for n, feafile in enumerate(args.feafile): print('Loading data from ', feafile, ' and ', args.capfile) enc_hsize = args.enc_hsize[n] enc_psize = args.enc_psize[n] feature_data = dh.load(feafile, args.capfile, vocabfile=args.vocabfile, vocab=vocab) feature_data = dh.check_feature_shape(feature_data) n_features = len(feature_data) data.extend(feature_data) enc_hsizes.extend([enc_hsize] * n_features) enc_psizes.extend([enc_psize] * n_features) data_sizes = map(lambda d: d["feature"].values()[0].shape[-1], data) logging.warn("The detectected feature lengths are: {}".format(data_sizes)) # A workaroung to reduce the lines to change later. args.in_size = data_sizes args.enc_hsize = enc_hsizes args.enc_psize = enc_psizes
vocab, train_args = pickle.load(f) model = torch.load(args.model + '.pth.tar') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) if train_args.dictmap != '': dictmap = json.load(open(train_args.dictmap, 'r')) else: dictmap = None # report data summary logging.info('#vocab = %d' % len(vocab)) # prepare test data logging.info('Loading test data from ' + args.test_set) test_data = dh.load(train_args.fea_type, args.test_path, args.test_set, vocab=vocab, dictmap=dictmap, include_caption=train_args.include_caption) test_indices, test_samples = dh.make_batch_indices(test_data, 1) logging.info('#test sample = %d' % test_samples) # generate sentences logging.info('-----------------------generate--------------------------') start_time = time.time() result = generate_response(model, test_data, test_indices, vocab, maxlen=args.maxlen, beam=args.beam, penalty=args.penalty, nbest=args.nbest)
print('Checking current datetime for correct election time') now = datetime.now() if now.weekday() == 5: #if now is Saturday if now.hour == 20: #if the hour is 8 pm if now.minute == 0: #if its 8:00pm print('Correct election time!') await election_cycle() sleep_time = 60 + ( 10 - now.second ) ## this makes the sleep function stick around the 10-second mark for the next call await asyncio.sleep(sleep_time) #load config print(f'Loading config...') if not config.load(): print('Could not load config') quit() print('\tDone') #load election data print(f'Loading election data...') if not data.load(): print('Could nto load election data') quit() print('\tDone') d_bot.loop.create_task(election_coroutine()) d_bot.run(config.get('discord_token'))
'%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s') logging.info('Command line: ' + ' '.join(sys.argv)) # get vocabulary logging.info('Extracting words from ' + args.train_set) vocab = dh.get_vocabulary(args.train_set, include_caption=args.include_caption) # load data logging.info('Loading training data from ' + args.train_set) train_data = dh.load(args.fea_type, args.train_path, args.train_set, vocabfile=args.vocabfile, include_caption=args.include_caption, vocab=vocab, dictmap=dictmap) logging.info('Loading validation data from ' + args.valid_set) valid_data = dh.load(args.fea_type, args.valid_path, args.valid_set, vocabfile=args.vocabfile, include_caption=args.include_caption, vocab=vocab, dictmap=dictmap) feature_dims, spatial_dims = dh.feature_shape(train_data) logging.info("Detected feature dims: {}".format(feature_dims))
logging.info('Loading model params from ' + args.model) path = args.model_conf with open(path, 'rb') as f: vocab, train_args = pickle.load(f) model = torch.load(args.model + '.pth.tar') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) # report data summary logging.info('#vocab = %d' % len(vocab)) # prepare test data logging.info('Loading test data from ' + args.test_set) test_data = dh.load(train_args.fea_type, args.test_path, args.test_set, vocab=vocab, include_caption=train_args.include_caption, separate_caption=train_args.separate_caption, max_history_length=train_args.max_history_length, merge_source=train_args.merge_source, undisclosed_only=args.undisclosed_only) test_indices, test_samples = dh.make_batch_indices( test_data, 1, separate_caption=train_args.separate_caption) logging.info('#test sample = %d' % test_samples) # generate sentences logging.info('-----------------------generate--------------------------') start_time = time.time() labeled_test = None if args.undisclosed_only and args.labeled_test is not None: labeled_test = json.load(open(args.labeled_test, 'r')) result = generate_response(model, test_data,
bert_tokenizer = GPT2Tokenizer.from_pretrained( train_args.bert_model) elif 'transfo-xl' in train_args.bert_model: bert_tokenizer = TransfoXLTokenizer.from_pretrained( train_args.bert_model) else: bert_tokenizer = None logging.info('Loading test data from ' + args.test_set) test_data = dh.load(train_args.fea_type, args.test_path, args.test_set, vocab=vocab, dictmap=dictmap, include_caption=train_args.include_caption, separate_caption=train_args.separate_caption, pretrained_elmo=train_args.pretrained_elmo, pretrained_bert=train_args.pretrained_bert, tokenizer=bert_tokenizer, bert_model=train_args.bert_model, pretrained_all=train_args.pretrained_all, concat_his=train_args.concat_his) test_indices, test_samples = dh.make_batch_indices( test_data, 1, separate_caption=train_args.separate_caption) logging.info('#test sample = %d' % test_samples) # generate sentences logging.info('-----------------------generate--------------------------') start_time = time.time() result = generate_response(model, test_data, test_indices,
else: bert_tokenizer = None logging.info('Extracting words from ' + args.train_set) vocab = dh.get_vocabulary(args.train_set, include_caption=args.include_caption, tokenizer=bert_tokenizer) if args.pretrained_word_emb is not None and 'none' not in args.pretrained_word_emb: pretrained_word_emb = dh.get_word_emb(vocab, args.pretrained_word_emb) else: pretrained_word_emb = None # load data logging.info('Loading training data from ' + args.train_set) train_data = dh.load(args.fea_type, args.train_path, args.train_set, vocabfile=args.vocabfile, include_caption=args.include_caption, separate_caption=args.separate_caption, vocab=vocab, dictmap=dictmap, pretrained_elmo=args.pretrained_elmo, pretrained_bert=args.pretrained_bert, bert_model=args.bert_model, tokenizer=bert_tokenizer, pretrained_all=args.pretrained_all, concat_his=args.concat_his) logging.info('Loading validation data from ' + args.valid_set) valid_data = dh.load(args.fea_type, args.valid_path, args.valid_set, vocabfile=args.vocabfile, include_caption=args.include_caption, separate_caption=args.separate_caption, vocab=vocab, dictmap=dictmap, pretrained_elmo=args.pretrained_elmo, pretrained_bert=args.pretrained_bert, bert_model=args.bert_model, tokenizer=bert_tokenizer, pretrained_all=args.pretrained_all, concat_his=args.concat_his) feature_dims = dh.feature_shape(train_data) logging.info("Detected feature dims: {}".format(feature_dims));