def evaluate(eval_ops, iterator, num_steps, num_examples, model_path=None, num_gpus=1, sess=None): #timer = gezi.Timer('evaluate') if model_path: ids_list = [] predictions_list = [] labels_list = [] losses = [] top_preds_list = [] if not sess: sess = melt.get_session() # for prcurve sess.run(iterator.initializer) for _ in range(num_steps): results = sess.run(eval_ops) for i in range(num_gpus): ids, loss, predictions, top_preds, labels = results[i] ids = gezi.decode(ids) #images = images.astype(np.uint8) losses.append(loss) predictions_list.append(predictions) top_preds_list.append(top_preds) labels_list.append(labels) if model_path: ids_list.append(ids) # notice loss might be not so accurate due to final batch padding but that's not big problem loss = np.mean(losses) if model_path: ids = np.concatenate(ids_list)[:num_examples] predicts = np.concatenate(predictions_list)[:num_examples] top_preds = np.concatenate(top_preds_list)[:num_examples] labels = np.concatenate(labels_list)[:num_examples] acc = np.mean(np.equal(predicts, labels)) results = [loss, acc] names = ['metric/valid/loss', 'metric/valid/acc'] if model_path: write(ids, predicts, model_path, labels, suffix='valid_info') #timer.print() #print(len(predicts)) return results, names
def deal(dataset, infos): for x, _ in tqdm(dataset, ascii=True): for key in x: x[key] = x[key].numpy() if type(x[key][0]) == bytes: x[key] = gezi.decode(x[key]) ids = x['id'] for j in range(len(ids)): infos[ids[j]] = {} for key in x: infos[ids[j]][key] = x[key][j]
def evaluate(model, dataset, eval_fn, model_path=None, names=None, write_fn=None, write_streaming=False, num_steps_per_epoch=None, suffix='.valid', sep=','): if FLAGS.torch: model.eval() if not write_fn: write_streaming = True predicts_list = [] labels_list = [] ids_list = [] ofile = model_path + suffix if model_path else None if write_streaming: out = open(ofile, 'w', encoding='utf-8') if ofile else None if out: if names is not None: print(*names, sep=sep, file=out) else: out = None for x, y in tqdm(dataset, total=num_steps_per_epoch, ascii=True): if FLAGS.torch: x, y = to_torch(x, y) predicts = model(x) if FLAGS.torch: predicts = predicts.detach().cpu() y = y.detach().cpu() predicts_list.append(predicts) labels_list.append(y) if not FLAGS.torch: ids = gezi.decode(x['id'].numpy()) else: ids = gezi.decode(x['id']) ids_list.append(ids) if out: for id, label, predict in zip(ids, y.numpy(), predicts.numpy()): if write_fn is None: if not gezi.iterable(label): label = [label] if not gezi.iterable(predict): predict = [predict] print(id, *label, *predict, sep=sep, file=out) else: write_fn(id, label, predict, out) # if FLAGS.torch: # predicts_list = [x.detach().numpy() for x in predicts_list] # labels_lis = [x.detach().numpy() for x in labels_list] predicts = np.concatenate(predicts_list) labels = np.concatenate(labels_list) ids = np.concatenate(ids_list) if out: out.close() if not write_streaming and ofile: write_fn(ids, labels, predicts, ofile) if len(inspect.getargspec(eval_fn).args) == 4: vals, names = eval_fn(labels, predicts, ids=ids, model_path=model_path) elif len(inspect.getargspec(eval_fn).args) == 3: if 'ids' in inspect.getargspec(eval_fn).args: vals, names = eval_fn(labels, predicts, ids) else: vals, names = eval_fn(labels, predicts) if model_path: with open(model_path + '.valid.metrics', 'w') as out: for val, name in zip(vals, names): print(name, val, sep='\t', file=out) return vals, names
def inference(model, dataset, model_path, names=None, debug_names=None, write_fn=None, write_streaming=False, num_steps_per_epoch=None, suffix='.infer', sep=','): if FLAGS.torch: model.eval() if not write_fn: write_streaming = True ofile = model_path + suffix ofile2 = ofile + '.debug' if write_streaming: if write_fn and len(inspect.getargspec(write_fn).args) == 4: out_debug = open(ofile2, 'w', encoding='utf-8') else: out_debug = None out = open(ofile, 'w', encoding='utf-8') else: out = None out_debug = None if write_streaming: if names is not None: print(*names, sep=sep, file=out) if debug_names and out_debug: print(*debug_names, sep=sep, file=out_debug) predicts_list = [] ids_list = [] for (x, _) in tqdm(dataset, total=num_steps_per_epoch, ascii=True): if FLAGS.torch: x = to_torch(x) predicts = model(x) if FLAGS.torch: predicts = predicts.detach().cpu() # here id is str in py3 will be bytes if not FLAGS.torch: ids = gezi.decode(x['id'].numpy()) else: ids = gezi.decode(x['id']) if not write_streaming: predicts_list.append(predicts) ids_list.append(ids) else: for id, predict in zip(ids, predicts.numpy()): if write_fn is None: if not gezi.iterable(predict): predict = [predict] print(id, *predict, sep=sep, file=out) else: if out_debug: write_fn(id, predict, out, out_debug) else: write_fn(id, predict, out) if out: out.close() if out_debug: out_debug.close() if not write_streaming: # if FLAGS.torch: # predicts_list = [x.detach().numpy() for x in predicts_list] predicts = np.concatenate(predicts_list) ids = np.concatenate(ids_list) if len(inspect.getargspec(write_fn).args) == 4: write_fn(ids, predicts, ofile, ofile2) else: write_fn(ids, predicts, ofile)
def main(_): in_dir = sys.argv[1] files = gezi.list_files(in_dir) total = melt.get_num_records(files) print('total', total, file=sys.stderr) if not total: exit(1) ofile = sys.argv[2] df = None if ofile and gezi.non_empty(os.path.realpath(ofile)): try: df = pd.read_csv(ofile) if len(df) == total and (not FLAGS.title or 'title' in df.columns): print(f'infos file {ofile} exits do nothing', file=sys.stderr) exit(0) else: print('num_done:', len(df), file=sys.stderr) except Exception: pass print('write to', ofile, file=sys.stderr) FLAGS.batch_size = FLAGS.batch_size_ batch_size = FLAGS.batch_size if tf.__version__ < '2': tf.compat.v1.enable_eager_execution() dataset = Dataset('valid') print('---batch_size', dataset.batch_size, FLAGS.batch_size, melt.batch_size(), file=sys.stderr) batches = dataset.make_batch(batch_size=batch_size, filenames=files, repeat=False) num_steps = -int(-total // batch_size) print('----num_steps', num_steps, file=sys.stderr) m = defaultdict(list) for i, (x, _) in tqdm(enumerate(batches), total=num_steps, ascii=True, desc='loop'): bs = len(x['id']) keys = list(x.keys()) for key in keys: x[key] = x[key].numpy() if not len(x[key]): del x[key] continue if x[key].shape == (bs, 1): x[key] = gezi.squeeze(x[key]) if x[key].shape != (bs,): del x[key] continue if x[key].dtype == np.object: x[key] = gezi.decode(x[key]) m[key] += [x[key]] if i == 0: if df is not None and len(df) == total and set(m.keys()) == set(list(df.columns)): print(f'infos file {ofile} exits do nothing', file=sys.stderr) exit(0) for key in m.keys(): m[key] = np.concatenate(m[key], 0) df = pd.DataFrame(m) df.to_csv(ofile, index=False)
def main(_): base = FLAGS.base logging.set_logging_path('./mount/tmp/') vocab_path = f'{base}/vocab.txt' ids2text.init(vocab_path) FLAGS.vocab = f'{base}/vocab.txt' # FLAGS.length_index = 2 # FLAGS.buckets = '100,400' # FLAGS.batch_sizes = '64,64,32' input_ = FLAGS.input if FLAGS.type == 'test': input_ = input_.replace('valid', 'test') inputs = gezi.list_files(input_) inputs.sort() if FLAGS.fold is not None: inputs = [x for x in inputs if not x.endswith('%d.record' % FLAGS.fold)] if FLAGS.type == 'debug': print('type', FLAGS.type, 'inputs', inputs, file=sys.stderr) dataset = Dataset('valid') dataset = dataset.make_batch(FLAGS.batch_size_, inputs) print('dataset', dataset) timer = gezi.Timer('read record') for i, (x, y) in enumerate(dataset): # if i % 10 == 1: # print(x['id']) # print(x['content'][0]) # print(ids2text.ids2text(x['content'][0], sep='|')) # print(x['content']) # print(type(x['id'].numpy()[0]) == bytes) # break x['id'] = gezi.decode(x['id'].numpy()) x['content_str'] = gezi.decode(x['content_str'].numpy()) for j, id in enumerate(x['id']): if id == '573': print(id, x['content_str'][j]) elif FLAGS.type == 'dump': valid_infos = {} test_infos = {} # TODO notice train and valid also share ids.. so valid only save 0 is ok... # 120000 doc but first 15000 train duplicate id with valid so only save valid result for those ids currently inputs = gezi.list_files(f'{base}/train/*record') dataset = Dataset('valid') dataset = dataset.make_batch(1, inputs) deal(dataset, valid_infos) print('after valid', len(valid_infos)) for key in valid_infos: print(valid_infos[key]) print(ids2text.ids2text(valid_infos[key]['content'])) break ofile = f'{base}/info.pkl' with open(ofile, 'wb') as out: pickle.dump(valid_infos, out) del valid_infos inputs = gezi.list_files(f'{base}/test/*record') dataset = Dataset('test') dataset = dataset.make_batch(1, inputs) deal(dataset, test_infos) print('after test', len(test_infos)) ofile = ofile.replace('.pkl', '.test.pkl') with open(ofile, 'wb') as out: pickle.dump(test_infos, out) for key in test_infos: print(test_infos[key]) print(ids2text.ids2text(test_infos[key]['content'])) break elif FLAGS.type == 'show_info': valid_infos = pickle.load(open(f'{base}/info.pkl', 'rb')) lens = [len(valid_infos[key]['content']) for key in valid_infos] unks = [list(valid_infos[key]['content']).count(FLAGS.unk_id) for key in valid_infos] print('num unks per doc:', sum(unks) / len(valid_infos)) print('num doc with unk ratio:', len([x for x in unks if x != 0]) / len(unks)) print('un unk tokens ratio:', sum(unks) / sum(lens)) print('len max:', np.max(lens)) print('len min:', np.min(lens)) print('len mean:', np.mean(lens)) print('num docs:', len(valid_infos)) num_show = 0 for key in valid_infos: if list(valid_infos[key]['content']).count(FLAGS.unk_id) > 0: print(valid_infos[key]) print(ids2text.ids2text(valid_infos[key]['content'])) num_show += 1 if num_show > 5: break del valid_infos print('--------------for test info:') test_infos = pickle.load(open(f'{base}/info.test.pkl', 'rb')) lens = [len(test_infos[key]['content']) for key in test_infos] unks = [list(test_infos[key]['content']).count(FLAGS.unk_id) for key in test_infos] print('num unks per doc:', sum(unks) / len(test_infos)) print('num doc with unk ratio:', len([x for x in unks if x != 0]) / len(test_infos)) print('un unk tokens ratio:', sum(unks) / sum(lens)) print('len max:', np.max(lens)) print('len min:', np.min(lens)) print('len mean:', np.mean(lens)) print('num docs:', len(test_infos)) num_show = 0 for key in test_infos: if list(test_infos[key]['content']).count(FLAGS.unk_id) > 0: print(test_infos[key]) print(ids2text.ids2text(test_infos[key]['content'])) num_show += 1 if num_show > 5: break else: raise ValueError(FLAGS.type)
def evaluate(model, dataset, eval_fn, model_path=None, names=None, write_fn=None, write_streaming=False, num_steps=None, num_examples=None, suffix='.valid', sep=','): if hasattr(model, 'eval'): model.eval() if not write_fn: write_streaming = True predicts_list = [] labels_list = [] ids_list = [] ofile = model_path + suffix if model_path else None if write_streaming: out = open(ofile, 'w', encoding='utf-8') if ofile else None if out: if names is not None: print(*names, sep=sep, file=out) else: out = None for x, y in tqdm(dataset, total=num_steps, ascii=True): if FLAGS.torch: x, y = to_torch(x, y) predicts = model(x) if FLAGS.torch: predicts = predicts.detach().cpu() y = y.detach().cpu() predicts_list.append(predicts) labels_list.append(y) if not FLAGS.torch: ids = gezi.decode(x['id'].numpy()) else: ids = gezi.decode(x['id']) ids_list.append(ids) if out: for id, label, predict in zip(ids, y.numpy(), predicts.numpy()): if write_fn is None: if not gezi.iterable(label): label = [label] if not gezi.iterable(predict): predict = [predict] print(id, *label, *predict, sep=sep, file=out) else: write_fn(id, label, predict, out) if out: out.close() # if FLAGS.torch: # predicts_list = [x.detach().numpy() for x in predicts_list] # labels_lis = [x.detach().numpy() for x in labels_list] if FLAGS.use_horovod and FLAGS.horovod_eval: #import horovod.torch as hvd #print('----------------------before hvd reduce') # TODO check eager mode ok... tensor = tf.constant(0) if not FLAGS.torch else torch.zeros(0) hvd.allreduce(tensor) ## here for horovod mutliple gpu dataset is not repeat mode ids_list = comm.allgather(np.concatenate(ids_list)) predicts_list = comm.allgather(np.concatenate(predicts_list)) labels_list = comm.allgather(np.concatenate(labels_list)) comm.barrier() ids2 = np.concatenate(ids_list) predicts2 = np.concatenate(predicts_list) labels2 = np.concatenate(labels_list) #----below is for batch parse which if not repeat mode then final batch will still same size not smaller # and not use repeat mode so last batch fill with id '' empty we can remove here ids = [] predicts = [] labels = [] for i in range(len(ids2)): if not ids2[i] == '': ids.append(ids2[i]) predicts.append(predicts2[i]) labels.append(labels2[i]) ids = np.array(ids) predicts = np.array(predicts) labels = np.array(labels) else: try: # concat list so like [[512,], [512,]...] -> [512 * num_batchs] # ore [[512, 3], [512,3] ..] -> [512 * num_batchs, 3] ids = np.concatenate(ids_list)[:num_examples] except Exception: ids = ['0'] * num_examples predicts = np.concatenate(predicts_list)[:num_examples] labels = np.concatenate(labels_list)[:num_examples] if not write_streaming and ofile and (not FLAGS.use_horovod or hvd.rank() == 0): write_fn(ids, labels, predicts, ofile) if len(inspect.getargspec(eval_fn).args) == 4: vals, names = eval_fn(labels, predicts, ids=ids, model_path=model_path) elif len(inspect.getargspec(eval_fn).args) == 3: if 'ids' in inspect.getargspec(eval_fn).args: vals, names = eval_fn(labels, predicts, ids) else: vals, names = eval_fn(labels, predicts, model_path) else: vals, names = eval_fn(labels, predicts) if model_path and (not FLAGS.use_horovod or hvd.rank() == 0): with open(model_path + '.valid.metrics', 'w') as out: for val, name in zip(vals, names): print(name, val, sep='\t', file=out) return vals, names
def inference(model, dataset, model_path, names=None, debug_names=None, write_fn=None, write_streaming=False, num_steps=None, num_examples=None, suffix='.infer', sep=','): if has_attr(model, 'eval'): model.eval() if not write_fn: write_streaming = True ofile = model_path + suffix ofile2 = ofile + '.debug' if write_streaming: if write_fn and len(inspect.getargspec(write_fn).args) == 4: out_debug = open(ofile2, 'w', encoding='utf-8') else: out_debug = None out = open(ofile, 'w', encoding='utf-8') else: out = None out_debug = None if write_streaming: if names is not None: print(*names, sep=sep, file=out) if debug_names and out_debug: print(*debug_names, sep=sep, file=out_debug) predicts_list = [] ids_list = [] for (x, _) in tqdm(dataset, total=num_steps, ascii=True): if FLAGS.torch: x = to_torch(x) predicts = model(x) if FLAGS.torch: predicts = predicts.detach().cpu() # here id is str in py3 will be bytes if not FLAGS.torch: ids = gezi.decode(x['id'].numpy()) else: ids = gezi.decode(x['id']) if not write_streaming: predicts_list.append(predicts) ids_list.append(ids) else: for id, predict in zip(ids, predicts.numpy()): if write_fn is None: if not gezi.iterable(predict): predict = [predict] print(id, *predict, sep=sep, file=out) else: if out_debug: write_fn(id, predict, out, out_debug) else: write_fn(id, predict, out) if out: out.close() if out_debug: out_debug.close() if not write_streaming: if FLAGS.use_horovod and FLAGS.horovod_eval: #import horovod.torch as hvd #print('----------------------before hvd reduce') tensor = tf.constant(0) if not FLAGS.torch else torch.zeros(0) hvd.allreduce(tensor) ## here for horovod mutliple gpu dataset is not repeat mode ids_list = comm.allgather(np.concatenate(ids_list)) predicts_list = comm.allgather(np.concatenate(predicts_list)) comm.barrier() ids2 = np.concatenate(ids_list) predicts2 = np.concatenate(predicts_list) #----below is for batch parse which if not repeat mode then final batch will still same size not smaller # and not use repeat mode so last batch fill with id '' empty we can remove here ids = [] predicts = [] for i in range(len(ids2)): if not ids2[i] == '': ids.append(ids2[i]) predicts.append(predicts2[i]) ids = np.array(ids) predicts = np.array(predicts) else: try: # concat list so like [[512,], [512,]...] -> [512 * num_batchs] # ore [[512, 3], [512,3] ..] -> [512 * num_batchs, 3] ids = np.concatenate(ids_list)[:num_examples] except Exception: ids = ['0'] * num_examples predicts = np.concatenate(predicts_list)[:num_examples] if (not FLAGS.use_horovod or hvd.rank() == 0): if len(inspect.getargspec(write_fn).args) == 4: write_fn(ids, predicts, ofile, ofile2) else: write_fn(ids, predicts, ofile)