def evaluate(eval_ops,
iterator,
num_steps,
num_examples,
model_path=None,
num_gpus=1,
sess=None):
#timer = gezi.Timer('evaluate')
if model_path:
ids_list = []
predictions_list = []
labels_list = []
losses = []
top_preds_list = []
if not sess:
sess = melt.get_session()
# for prcurve
sess.run(iterator.initializer)
for _ in range(num_steps):
results = sess.run(eval_ops)
for i in range(num_gpus):
ids, loss, predictions, top_preds, labels = results[i]
ids = gezi.decode(ids)
#images = images.astype(np.uint8)
losses.append(loss)
predictions_list.append(predictions)
top_preds_list.append(top_preds)
labels_list.append(labels)
if model_path:
ids_list.append(ids)
# notice loss might be not so accurate due to final batch padding but that's not big problem
loss = np.mean(losses)
if model_path:
ids = np.concatenate(ids_list)[:num_examples]
predicts = np.concatenate(predictions_list)[:num_examples]
top_preds = np.concatenate(top_preds_list)[:num_examples]
labels = np.concatenate(labels_list)[:num_examples]
acc = np.mean(np.equal(predicts, labels))
results = [loss, acc]
names = ['metric/valid/loss', 'metric/valid/acc']
if model_path:
write(ids, predicts, model_path, labels, suffix='valid_info')
#timer.print()
#print(len(predicts))
return results, names
def deal(dataset, infos):
for x, _ in tqdm(dataset, ascii=True):
for key in x:
x[key] = x[key].numpy()
if type(x[key][0]) == bytes:
x[key] = gezi.decode(x[key])
ids = x['id']
for j in range(len(ids)):
infos[ids[j]] = {}
for key in x:
infos[ids[j]][key] = x[key][j]
def evaluate(model,
dataset,
eval_fn,
model_path=None,
names=None,
write_fn=None,
write_streaming=False,
num_steps_per_epoch=None,
suffix='.valid',
sep=','):
if FLAGS.torch:
model.eval()
if not write_fn:
write_streaming = True
predicts_list = []
labels_list = []
ids_list = []
ofile = model_path + suffix if model_path else None
if write_streaming:
out = open(ofile, 'w', encoding='utf-8') if ofile else None
if out:
if names is not None:
print(*names, sep=sep, file=out)
else:
out = None
for x, y in tqdm(dataset, total=num_steps_per_epoch, ascii=True):
if FLAGS.torch:
x, y = to_torch(x, y)
predicts = model(x)
if FLAGS.torch:
predicts = predicts.detach().cpu()
y = y.detach().cpu()
predicts_list.append(predicts)
labels_list.append(y)
if not FLAGS.torch:
ids = gezi.decode(x['id'].numpy())
else:
ids = gezi.decode(x['id'])
ids_list.append(ids)
if out:
for id, label, predict in zip(ids, y.numpy(), predicts.numpy()):
if write_fn is None:
if not gezi.iterable(label):
label = [label]
if not gezi.iterable(predict):
predict = [predict]
print(id, *label, *predict, sep=sep, file=out)
else:
write_fn(id, label, predict, out)
# if FLAGS.torch:
# predicts_list = [x.detach().numpy() for x in predicts_list]
# labels_lis = [x.detach().numpy() for x in labels_list]
predicts = np.concatenate(predicts_list)
labels = np.concatenate(labels_list)
ids = np.concatenate(ids_list)
if out:
out.close()
if not write_streaming and ofile:
write_fn(ids, labels, predicts, ofile)
if len(inspect.getargspec(eval_fn).args) == 4:
vals, names = eval_fn(labels, predicts, ids=ids, model_path=model_path)
elif len(inspect.getargspec(eval_fn).args) == 3:
if 'ids' in inspect.getargspec(eval_fn).args:
vals, names = eval_fn(labels, predicts, ids)
else:
vals, names = eval_fn(labels, predicts)
if model_path:
with open(model_path + '.valid.metrics', 'w') as out:
for val, name in zip(vals, names):
print(name, val, sep='\t', file=out)
return vals, names
def inference(model,
dataset,
model_path,
names=None,
debug_names=None,
write_fn=None,
write_streaming=False,
num_steps_per_epoch=None,
suffix='.infer',
sep=','):
if FLAGS.torch:
model.eval()
if not write_fn:
write_streaming = True
ofile = model_path + suffix
ofile2 = ofile + '.debug'
if write_streaming:
if write_fn and len(inspect.getargspec(write_fn).args) == 4:
out_debug = open(ofile2, 'w', encoding='utf-8')
else:
out_debug = None
out = open(ofile, 'w', encoding='utf-8')
else:
out = None
out_debug = None
if write_streaming:
if names is not None:
print(*names, sep=sep, file=out)
if debug_names and out_debug:
print(*debug_names, sep=sep, file=out_debug)
predicts_list = []
ids_list = []
for (x, _) in tqdm(dataset, total=num_steps_per_epoch, ascii=True):
if FLAGS.torch:
x = to_torch(x)
predicts = model(x)
if FLAGS.torch:
predicts = predicts.detach().cpu()
# here id is str in py3 will be bytes
if not FLAGS.torch:
ids = gezi.decode(x['id'].numpy())
else:
ids = gezi.decode(x['id'])
if not write_streaming:
predicts_list.append(predicts)
ids_list.append(ids)
else:
for id, predict in zip(ids, predicts.numpy()):
if write_fn is None:
if not gezi.iterable(predict):
predict = [predict]
print(id, *predict, sep=sep, file=out)
else:
if out_debug:
write_fn(id, predict, out, out_debug)
else:
write_fn(id, predict, out)
if out:
out.close()
if out_debug:
out_debug.close()
if not write_streaming:
# if FLAGS.torch:
# predicts_list = [x.detach().numpy() for x in predicts_list]
predicts = np.concatenate(predicts_list)
ids = np.concatenate(ids_list)
if len(inspect.getargspec(write_fn).args) == 4:
write_fn(ids, predicts, ofile, ofile2)
else:
write_fn(ids, predicts, ofile)
def main(_):
in_dir = sys.argv[1]
files = gezi.list_files(in_dir)
total = melt.get_num_records(files)
print('total', total, file=sys.stderr)
if not total:
exit(1)
ofile = sys.argv[2]
df = None
if ofile and gezi.non_empty(os.path.realpath(ofile)):
try:
df = pd.read_csv(ofile)
if len(df) == total and (not FLAGS.title or 'title' in df.columns):
print(f'infos file {ofile} exits do nothing', file=sys.stderr)
exit(0)
else:
print('num_done:', len(df), file=sys.stderr)
except Exception:
pass
print('write to', ofile, file=sys.stderr)
FLAGS.batch_size = FLAGS.batch_size_
batch_size = FLAGS.batch_size
if tf.__version__ < '2':
tf.compat.v1.enable_eager_execution()
dataset = Dataset('valid')
print('---batch_size', dataset.batch_size, FLAGS.batch_size, melt.batch_size(), file=sys.stderr)
batches = dataset.make_batch(batch_size=batch_size, filenames=files, repeat=False)
num_steps = -int(-total // batch_size)
print('----num_steps', num_steps, file=sys.stderr)
m = defaultdict(list)
for i, (x, _) in tqdm(enumerate(batches), total=num_steps, ascii=True, desc='loop'):
bs = len(x['id'])
keys = list(x.keys())
for key in keys:
x[key] = x[key].numpy()
if not len(x[key]):
del x[key]
continue
if x[key].shape == (bs, 1):
x[key] = gezi.squeeze(x[key])
if x[key].shape != (bs,):
del x[key]
continue
if x[key].dtype == np.object:
x[key] = gezi.decode(x[key])
m[key] += [x[key]]
if i == 0:
if df is not None and len(df) == total and set(m.keys()) == set(list(df.columns)):
print(f'infos file {ofile} exits do nothing', file=sys.stderr)
exit(0)
for key in m.keys():
m[key] = np.concatenate(m[key], 0)
df = pd.DataFrame(m)
df.to_csv(ofile, index=False)
def main(_):
base = FLAGS.base
logging.set_logging_path('./mount/tmp/')
vocab_path = f'{base}/vocab.txt'
ids2text.init(vocab_path)
FLAGS.vocab = f'{base}/vocab.txt'
# FLAGS.length_index = 2
# FLAGS.buckets = '100,400'
# FLAGS.batch_sizes = '64,64,32'
input_ = FLAGS.input
if FLAGS.type == 'test':
input_ = input_.replace('valid', 'test')
inputs = gezi.list_files(input_)
inputs.sort()
if FLAGS.fold is not None:
inputs = [x for x in inputs if not x.endswith('%d.record' % FLAGS.fold)]
if FLAGS.type == 'debug':
print('type', FLAGS.type, 'inputs', inputs, file=sys.stderr)
dataset = Dataset('valid')
dataset = dataset.make_batch(FLAGS.batch_size_, inputs)
print('dataset', dataset)
timer = gezi.Timer('read record')
for i, (x, y) in enumerate(dataset):
# if i % 10 == 1:
# print(x['id'])
# print(x['content'][0])
# print(ids2text.ids2text(x['content'][0], sep='|'))
# print(x['content'])
# print(type(x['id'].numpy()[0]) == bytes)
# break
x['id'] = gezi.decode(x['id'].numpy())
x['content_str'] = gezi.decode(x['content_str'].numpy())
for j, id in enumerate(x['id']):
if id == '573':
print(id, x['content_str'][j])
elif FLAGS.type == 'dump':
valid_infos = {}
test_infos = {}
# TODO notice train and valid also share ids.. so valid only save 0 is ok...
# 120000 doc but first 15000 train duplicate id with valid so only save valid result for those ids currently
inputs = gezi.list_files(f'{base}/train/*record')
dataset = Dataset('valid')
dataset = dataset.make_batch(1, inputs)
deal(dataset, valid_infos)
print('after valid', len(valid_infos))
for key in valid_infos:
print(valid_infos[key])
print(ids2text.ids2text(valid_infos[key]['content']))
break
ofile = f'{base}/info.pkl'
with open(ofile, 'wb') as out:
pickle.dump(valid_infos, out)
del valid_infos
inputs = gezi.list_files(f'{base}/test/*record')
dataset = Dataset('test')
dataset = dataset.make_batch(1, inputs)
deal(dataset, test_infos)
print('after test', len(test_infos))
ofile = ofile.replace('.pkl', '.test.pkl')
with open(ofile, 'wb') as out:
pickle.dump(test_infos, out)
for key in test_infos:
print(test_infos[key])
print(ids2text.ids2text(test_infos[key]['content']))
break
elif FLAGS.type == 'show_info':
valid_infos = pickle.load(open(f'{base}/info.pkl', 'rb'))
lens = [len(valid_infos[key]['content']) for key in valid_infos]
unks = [list(valid_infos[key]['content']).count(FLAGS.unk_id) for key in valid_infos]
print('num unks per doc:', sum(unks) / len(valid_infos))
print('num doc with unk ratio:', len([x for x in unks if x != 0]) / len(unks))
print('un unk tokens ratio:', sum(unks) / sum(lens))
print('len max:', np.max(lens))
print('len min:', np.min(lens))
print('len mean:', np.mean(lens))
print('num docs:', len(valid_infos))
num_show = 0
for key in valid_infos:
if list(valid_infos[key]['content']).count(FLAGS.unk_id) > 0:
print(valid_infos[key])
print(ids2text.ids2text(valid_infos[key]['content']))
num_show += 1
if num_show > 5:
break
del valid_infos
print('--------------for test info:')
test_infos = pickle.load(open(f'{base}/info.test.pkl', 'rb'))
lens = [len(test_infos[key]['content']) for key in test_infos]
unks = [list(test_infos[key]['content']).count(FLAGS.unk_id) for key in test_infos]
print('num unks per doc:', sum(unks) / len(test_infos))
print('num doc with unk ratio:', len([x for x in unks if x != 0]) / len(test_infos))
print('un unk tokens ratio:', sum(unks) / sum(lens))
print('len max:', np.max(lens))
print('len min:', np.min(lens))
print('len mean:', np.mean(lens))
print('num docs:', len(test_infos))
num_show = 0
for key in test_infos:
if list(test_infos[key]['content']).count(FLAGS.unk_id) > 0:
print(test_infos[key])
print(ids2text.ids2text(test_infos[key]['content']))
num_show += 1
if num_show > 5:
break
else:
raise ValueError(FLAGS.type)
def evaluate(model, dataset, eval_fn, model_path=None,
names=None, write_fn=None, write_streaming=False,
num_steps=None, num_examples=None,
suffix='.valid', sep=','):
if hasattr(model, 'eval'):
model.eval()
if not write_fn:
write_streaming = True
predicts_list = []
labels_list = []
ids_list = []
ofile = model_path + suffix if model_path else None
if write_streaming:
out = open(ofile, 'w', encoding='utf-8') if ofile else None
if out:
if names is not None:
print(*names, sep=sep, file=out)
else:
out = None
for x, y in tqdm(dataset, total=num_steps, ascii=True):
if FLAGS.torch:
x, y = to_torch(x, y)
predicts = model(x)
if FLAGS.torch:
predicts = predicts.detach().cpu()
y = y.detach().cpu()
predicts_list.append(predicts)
labels_list.append(y)
if not FLAGS.torch:
ids = gezi.decode(x['id'].numpy())
else:
ids = gezi.decode(x['id'])
ids_list.append(ids)
if out:
for id, label, predict in zip(ids, y.numpy(), predicts.numpy()):
if write_fn is None:
if not gezi.iterable(label):
label = [label]
if not gezi.iterable(predict):
predict = [predict]
print(id, *label, *predict, sep=sep, file=out)
else:
write_fn(id, label, predict, out)
if out:
out.close()
# if FLAGS.torch:
# predicts_list = [x.detach().numpy() for x in predicts_list]
# labels_lis = [x.detach().numpy() for x in labels_list]
if FLAGS.use_horovod and FLAGS.horovod_eval:
#import horovod.torch as hvd
#print('----------------------before hvd reduce')
# TODO check eager mode ok...
tensor = tf.constant(0) if not FLAGS.torch else torch.zeros(0)
hvd.allreduce(tensor)
## here for horovod mutliple gpu dataset is not repeat mode
ids_list = comm.allgather(np.concatenate(ids_list))
predicts_list = comm.allgather(np.concatenate(predicts_list))
labels_list = comm.allgather(np.concatenate(labels_list))
comm.barrier()
ids2 = np.concatenate(ids_list)
predicts2 = np.concatenate(predicts_list)
labels2 = np.concatenate(labels_list)
#----below is for batch parse which if not repeat mode then final batch will still same size not smaller
# and not use repeat mode so last batch fill with id '' empty we can remove here
ids = []
predicts = []
labels = []
for i in range(len(ids2)):
if not ids2[i] == '':
ids.append(ids2[i])
predicts.append(predicts2[i])
labels.append(labels2[i])
ids = np.array(ids)
predicts = np.array(predicts)
labels = np.array(labels)
else:
try:
# concat list so like [[512,], [512,]...] -> [512 * num_batchs]
# ore [[512, 3], [512,3] ..] -> [512 * num_batchs, 3]
ids = np.concatenate(ids_list)[:num_examples]
except Exception:
ids = ['0'] * num_examples
predicts = np.concatenate(predicts_list)[:num_examples]
labels = np.concatenate(labels_list)[:num_examples]
if not write_streaming and ofile and (not FLAGS.use_horovod or hvd.rank() == 0):
write_fn(ids, labels, predicts, ofile)
if len(inspect.getargspec(eval_fn).args) == 4:
vals, names = eval_fn(labels, predicts, ids=ids, model_path=model_path)
elif len(inspect.getargspec(eval_fn).args) == 3:
if 'ids' in inspect.getargspec(eval_fn).args:
vals, names = eval_fn(labels, predicts, ids)
else:
vals, names = eval_fn(labels, predicts, model_path)
else:
vals, names = eval_fn(labels, predicts)
if model_path and (not FLAGS.use_horovod or hvd.rank() == 0):
with open(model_path + '.valid.metrics', 'w') as out:
for val, name in zip(vals, names):
print(name, val, sep='\t', file=out)
return vals, names
def inference(model, dataset, model_path,
names=None, debug_names=None,
write_fn=None, write_streaming=False,
num_steps=None, num_examples=None,
suffix='.infer', sep=','):
if has_attr(model, 'eval'):
model.eval()
if not write_fn:
write_streaming = True
ofile = model_path + suffix
ofile2 = ofile + '.debug'
if write_streaming:
if write_fn and len(inspect.getargspec(write_fn).args) == 4:
out_debug = open(ofile2, 'w', encoding='utf-8')
else:
out_debug = None
out = open(ofile, 'w', encoding='utf-8')
else:
out = None
out_debug = None
if write_streaming:
if names is not None:
print(*names, sep=sep, file=out)
if debug_names and out_debug:
print(*debug_names, sep=sep, file=out_debug)
predicts_list = []
ids_list = []
for (x, _) in tqdm(dataset, total=num_steps, ascii=True):
if FLAGS.torch:
x = to_torch(x)
predicts = model(x)
if FLAGS.torch:
predicts = predicts.detach().cpu()
# here id is str in py3 will be bytes
if not FLAGS.torch:
ids = gezi.decode(x['id'].numpy())
else:
ids = gezi.decode(x['id'])
if not write_streaming:
predicts_list.append(predicts)
ids_list.append(ids)
else:
for id, predict in zip(ids, predicts.numpy()):
if write_fn is None:
if not gezi.iterable(predict):
predict = [predict]
print(id, *predict, sep=sep, file=out)
else:
if out_debug:
write_fn(id, predict, out, out_debug)
else:
write_fn(id, predict, out)
if out:
out.close()
if out_debug:
out_debug.close()
if not write_streaming:
if FLAGS.use_horovod and FLAGS.horovod_eval:
#import horovod.torch as hvd
#print('----------------------before hvd reduce')
tensor = tf.constant(0) if not FLAGS.torch else torch.zeros(0)
hvd.allreduce(tensor)
## here for horovod mutliple gpu dataset is not repeat mode
ids_list = comm.allgather(np.concatenate(ids_list))
predicts_list = comm.allgather(np.concatenate(predicts_list))
comm.barrier()
ids2 = np.concatenate(ids_list)
predicts2 = np.concatenate(predicts_list)
#----below is for batch parse which if not repeat mode then final batch will still same size not smaller
# and not use repeat mode so last batch fill with id '' empty we can remove here
ids = []
predicts = []
for i in range(len(ids2)):
if not ids2[i] == '':
ids.append(ids2[i])
predicts.append(predicts2[i])
ids = np.array(ids)
predicts = np.array(predicts)
else:
try:
# concat list so like [[512,], [512,]...] -> [512 * num_batchs]
# ore [[512, 3], [512,3] ..] -> [512 * num_batchs, 3]
ids = np.concatenate(ids_list)[:num_examples]
except Exception:
ids = ['0'] * num_examples
predicts = np.concatenate(predicts_list)[:num_examples]
if (not FLAGS.use_horovod or hvd.rank() == 0):
if len(inspect.getargspec(write_fn).args) == 4:
write_fn(ids, predicts, ofile, ofile2)
else:
write_fn(ids, predicts, ofile)
