np.random.seed(args.rand_seed)
if args.verbose >= 1:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s')
else:
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s: %(message)s')
for arg in vars(args):
print("{}={}".format(arg, getattr(args, arg)))
# get vocabulary
logging.info('Extracting words from ' + args.train_set)
vocab = dh.get_vocabulary(args.train_set, include_caption=args.include_caption)
# load data
logging.info('Loading training data from ' + args.train_set)
train_data = dh.load(args.fea_type, args.train_path, args.train_set,
include_caption=args.include_caption, separate_caption=args.separate_caption,
vocab=vocab, max_history_length=args.max_history_length,
merge_source=args.merge_source)
logging.info('Loading validation data from ' + args.valid_set)
valid_data = dh.load(args.fea_type, args.valid_path, args.valid_set,
include_caption=args.include_caption, separate_caption=args.separate_caption,
vocab=vocab, max_history_length=args.max_history_length,
merge_source=args.merge_source)
if args.fea_type[0] == 'none':
feature_dims = 0
else:
feature_dims = dh.feature_shape(train_data)
logging.info("Detected feature dims: {}".format(feature_dims));
# report data summary
logging.info('#vocab = %d' % len(vocab))
# make batchset for training
train_indices, train_samples = dh.make_batch_indices(train_data, args.batch_size,
# load data
vocab = {'<unk>': 0, '<sos>': 1, '<eos>': 2, '<no_tag>': 3}
data = []
enc_hsizes = []
enc_psizes = []
for a in range(0, 1):
feafile = args.feafile
print('Loading data from', feafile, 'and', args.capfile)
for n, feafile in enumerate(args.feafile):
print('Loading data from ', feafile, ' and ', args.capfile)
enc_hsize = args.enc_hsize[n]
enc_psize = args.enc_psize[n]
feature_data = dh.load(feafile,
args.capfile,
vocabfile=args.vocabfile,
vocab=vocab)
feature_data = dh.check_feature_shape(feature_data)
n_features = len(feature_data)
data.extend(feature_data)
enc_hsizes.extend([enc_hsize] * n_features)
enc_psizes.extend([enc_psize] * n_features)
data_sizes = map(lambda d: d["feature"].values()[0].shape[-1], data)
logging.warn("The detectected feature lengths are: {}".format(data_sizes))
# A workaroung to reduce the lines to change later.
args.in_size = data_sizes
args.enc_hsize = enc_hsizes
args.enc_psize = enc_psizes
vocab, train_args = pickle.load(f)
model = torch.load(args.model + '.pth.tar')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
if train_args.dictmap != '':
dictmap = json.load(open(train_args.dictmap, 'r'))
else:
dictmap = None
# report data summary
logging.info('#vocab = %d' % len(vocab))
# prepare test data
logging.info('Loading test data from ' + args.test_set)
test_data = dh.load(train_args.fea_type,
args.test_path,
args.test_set,
vocab=vocab,
dictmap=dictmap,
include_caption=train_args.include_caption)
test_indices, test_samples = dh.make_batch_indices(test_data, 1)
logging.info('#test sample = %d' % test_samples)
# generate sentences
logging.info('-----------------------generate--------------------------')
start_time = time.time()
result = generate_response(model,
test_data,
test_indices,
vocab,
maxlen=args.maxlen,
beam=args.beam,
penalty=args.penalty,
nbest=args.nbest)
print('Checking current datetime for correct election time')
now = datetime.now()
if now.weekday() == 5: #if now is Saturday
if now.hour == 20: #if the hour is 8 pm
if now.minute == 0: #if its 8:00pm
print('Correct election time!')
await election_cycle()
sleep_time = 60 + (
10 - now.second
) ## this makes the sleep function stick around the 10-second mark for the next call
await asyncio.sleep(sleep_time)
#load config
print(f'Loading config...')
if not config.load():
print('Could not load config')
quit()
print('\tDone')
#load election data
print(f'Loading election data...')
if not data.load():
print('Could nto load election data')
quit()
print('\tDone')
d_bot.loop.create_task(election_coroutine())
d_bot.run(config.get('discord_token'))
'%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s')
else:
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(levelname)s: %(message)s')
logging.info('Command line: ' + ' '.join(sys.argv))
# get vocabulary
logging.info('Extracting words from ' + args.train_set)
vocab = dh.get_vocabulary(args.train_set,
include_caption=args.include_caption)
# load data
logging.info('Loading training data from ' + args.train_set)
train_data = dh.load(args.fea_type,
args.train_path,
args.train_set,
vocabfile=args.vocabfile,
include_caption=args.include_caption,
vocab=vocab,
dictmap=dictmap)
logging.info('Loading validation data from ' + args.valid_set)
valid_data = dh.load(args.fea_type,
args.valid_path,
args.valid_set,
vocabfile=args.vocabfile,
include_caption=args.include_caption,
vocab=vocab,
dictmap=dictmap)
feature_dims, spatial_dims = dh.feature_shape(train_data)
logging.info("Detected feature dims: {}".format(feature_dims))
logging.info('Loading model params from ' + args.model)
path = args.model_conf
with open(path, 'rb') as f:
vocab, train_args = pickle.load(f)
model = torch.load(args.model + '.pth.tar')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
# report data summary
logging.info('#vocab = %d' % len(vocab))
# prepare test data
logging.info('Loading test data from ' + args.test_set)
test_data = dh.load(train_args.fea_type,
args.test_path,
args.test_set,
vocab=vocab,
include_caption=train_args.include_caption,
separate_caption=train_args.separate_caption,
max_history_length=train_args.max_history_length,
merge_source=train_args.merge_source,
undisclosed_only=args.undisclosed_only)
test_indices, test_samples = dh.make_batch_indices(
test_data, 1, separate_caption=train_args.separate_caption)
logging.info('#test sample = %d' % test_samples)
# generate sentences
logging.info('-----------------------generate--------------------------')
start_time = time.time()
labeled_test = None
if args.undisclosed_only and args.labeled_test is not None:
labeled_test = json.load(open(args.labeled_test, 'r'))
result = generate_response(model,
test_data,
bert_tokenizer = GPT2Tokenizer.from_pretrained(
train_args.bert_model)
elif 'transfo-xl' in train_args.bert_model:
bert_tokenizer = TransfoXLTokenizer.from_pretrained(
train_args.bert_model)
else:
bert_tokenizer = None
logging.info('Loading test data from ' + args.test_set)
test_data = dh.load(train_args.fea_type,
args.test_path,
args.test_set,
vocab=vocab,
dictmap=dictmap,
include_caption=train_args.include_caption,
separate_caption=train_args.separate_caption,
pretrained_elmo=train_args.pretrained_elmo,
pretrained_bert=train_args.pretrained_bert,
tokenizer=bert_tokenizer,
bert_model=train_args.bert_model,
pretrained_all=train_args.pretrained_all,
concat_his=train_args.concat_his)
test_indices, test_samples = dh.make_batch_indices(
test_data, 1, separate_caption=train_args.separate_caption)
logging.info('#test sample = %d' % test_samples)
# generate sentences
logging.info('-----------------------generate--------------------------')
start_time = time.time()
result = generate_response(model,
test_data,
test_indices,
else:
bert_tokenizer = None
logging.info('Extracting words from ' + args.train_set)
vocab = dh.get_vocabulary(args.train_set, include_caption=args.include_caption, tokenizer=bert_tokenizer)
if args.pretrained_word_emb is not None and 'none' not in args.pretrained_word_emb:
pretrained_word_emb = dh.get_word_emb(vocab, args.pretrained_word_emb)
else:
pretrained_word_emb = None
# load data
logging.info('Loading training data from ' + args.train_set)
train_data = dh.load(args.fea_type, args.train_path, args.train_set,
vocabfile=args.vocabfile,
include_caption=args.include_caption, separate_caption=args.separate_caption,
vocab=vocab, dictmap=dictmap,
pretrained_elmo=args.pretrained_elmo, pretrained_bert=args.pretrained_bert,
bert_model=args.bert_model, tokenizer=bert_tokenizer,
pretrained_all=args.pretrained_all, concat_his=args.concat_his)
logging.info('Loading validation data from ' + args.valid_set)
valid_data = dh.load(args.fea_type, args.valid_path, args.valid_set,
vocabfile=args.vocabfile,
include_caption=args.include_caption, separate_caption=args.separate_caption,
vocab=vocab, dictmap=dictmap,
pretrained_elmo=args.pretrained_elmo, pretrained_bert=args.pretrained_bert,
bert_model=args.bert_model, tokenizer=bert_tokenizer,
pretrained_all=args.pretrained_all, concat_his=args.concat_his)
feature_dims = dh.feature_shape(train_data)
logging.info("Detected feature dims: {}".format(feature_dims));
