def get_test_np(self):
if self.test_tfds is None:
error("Error: test_tfds is None.")
return self.accum_test_x, self.accum_test_y
if len(self.accum_test_x) == 0:
time_test_np_start = time.time()
tfds_test_os_iterator = self.test_tfds.make_one_shot_iterator()
as_timer("tfds_test_ositer")
tfds_test_iter_next = tfds_test_os_iterator.get_next()
# with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess:
if self.if_padded_batch:
while True:
try:
# example, labels = sess.run(tfds_test_iter_next)
example, labels = self.tfds_convertor_sess.run(
tfds_test_iter_next)
example = np.squeeze(example, (2, 3))
example = np.squeeze(example, axis=-1)
example = example.astype(np.int)
self.accum_test_x.extend(example)
self.accum_test_y.extend(labels)
self.accm_test_cnt += example.shape[0]
# X.append(example)
# Y.append(labels)
except tf.errors.OutOfRangeError:
break
else:
while True:
try:
# example, labels = sess.run(tfds_test_iter_next)
example, labels = self.tfds_convertor_sess.run(
tfds_test_iter_next)
# output: Note:time example shape=(86401, 1, 1, 1)
# info("Note:time example shape={}".format(example.shape))
self.accum_test_x.append(example)
self.accum_test_y.append(labels)
self.accm_test_cnt += 1
except tf.errors.OutOfRangeError:
as_timer("tfds_test_run_OOR_{}".format(
self.accm_test_cnt))
break
time_test_np_end = time.time()
info("note: now take test accm_test_cnt={}, cost_time={}s".format(
self.accm_test_cnt,
round(time_test_np_end - time_test_np_start, 3)))
self.accum_test_y = np.array(self.accum_test_y)
# return self.accum_test_x
return [np.squeeze(x) for x in self.accum_test_x]
def init_train_tfds(self, train_tfds, train_num, force_shuffle=False):
if self.train_tfds is None or self.train_num == 0 or force_shuffle is True:
self.train_num = train_num
self.train_tfds = train_tfds
if self.if_train_shuffle or force_shuffle is True:
self.train_tfds = self.train_tfds.shuffle(buffer_size=min(
self.train_max_shuffle_size, int(self.train_num * 0.6)))
# self.train_tfds = self.train_tfds.shuffle(buffer_size=self.train_shuffle_size)
if self.if_padded_batch:
self.train_tfds = self.train_tfds.padded_batch(
self.padded_batch_size,
padded_shapes=([None, 1, 1, 1], [None]),
padding_values=(tf.constant(-1, dtype=tf.float32),
tf.constant(-1, dtype=tf.float32)),
)
# force reNone tfds_train_iterator.
self.tfds_train_os_iterator = None
info(
"note: train_tfds cache, if_train_shuffle={}, force_shuffle={}, reset tfds_train_os_iterator None."
.format(self.if_train_shuffle, force_shuffle))
def get_train_np(self, take_size):
as_timer("tfdscvtr_get_train_np_start")
if self.train_tfds is None:
error("Error: train_tfds is None.")
return self.accum_train_x, self.accum_train_y
if self.tfds_train_os_iterator is None:
self.tfds_train_os_iterator = self.train_tfds.make_one_shot_iterator(
)
as_timer("tfds_train_os_iterator_make")
self.tfds_train_iter_next = self.tfds_train_os_iterator.get_next()
cur_get_cnt = 0
cur_data_y = list()
cur_incre_train_x = list()
if self.accm_train_cnt < self.train_num:
# info("note: accm_train_cnt={}, train_num={}".format(self.accm_train_cnt, self.train_num))
time_train_np_start = time.time()
if self.if_padded_batch:
info("note: domain={}".format(self.domain))
while True:
example_batch_num = 0
try:
example, labels = self.tfds_convertor_sess.run(
self.tfds_train_iter_next)
example = np.squeeze(example, (2, 3))
example = np.squeeze(example, axis=-1)
example = example.astype(np.int)
# fixme: 注意，这里example 和 labels都是batch, batch_size=20
cur_incre_train_x.extend(example)
cur_data_y.extend(labels)
# X.append(example)
# Y.append(labels)
cur_get_cnt += example.shape[0]
self.accm_train_cnt += example.shape[0]
example_batch_num += 1
# info("note: cur_get_cnt={}, accm_train_cnt={}, example_batch_num={}, a_example_shape={}".format(cur_get_cnt, self.accm_train_cnt, example_batch_num, example.shape))
if cur_get_cnt >= take_size or self.accm_train_cnt >= self.train_num:
time_train_np_end = time.time()
info(
"note: now text extend batch domain={} take train update={}, accm_train_cnt={}, cost_time={}s"
.format(
self.domain, cur_get_cnt,
self.accm_train_cnt,
round(
time_train_np_end -
time_train_np_start, 3)))
break
except tf.errors.OutOfRangeError:
info("train out of range, cur_get_cnt={}".format(
cur_get_cnt))
break
else:
while True:
try:
example, labels = self.tfds_convertor_sess.run(
self.tfds_train_iter_next)
# output: Note:time example shape=(86401, 1, 1, 1)
# logger.info("Note:time example shape={}".format(example.shape))
# self.accum_train_x.append(example)
cur_incre_train_x.append(example)
cur_data_y.append(labels)
cur_get_cnt += 1
self.accm_train_cnt += 1
if cur_get_cnt >= take_size or self.accm_train_cnt >= self.train_num:
time_train_np_end = time.time()
info(
"note: now append domain={} take train update={}, accm_train_cnt={}, train_num={}, cost_time={}s"
.format(
self.domain, cur_get_cnt,
self.accm_train_cnt, self.train_num,
round(
time_train_np_end -
time_train_np_start, 3)))
as_timer("tfds_get_train_np_update={}".format(
cur_get_cnt))
break
except tf.errors.OutOfRangeError:
break
self.accum_train_x.extend(cur_incre_train_x)
as_timer("tfds_get_train_np_accum_train_x_{}".format(
len(self.accum_train_x)))
if self.accum_train_y is None:
self.accum_train_y = np.array(cur_data_y)
else:
self.accum_train_y = np.concatenate(
(self.accum_train_y, np.array(cur_data_y)))
info(
"note: self.accum_train_x num_new={}, incre_train_num={}, self.accum_train_y shape={}, cur_data_y shape={}"
.format(len(self.accum_train_x), len(cur_incre_train_x),
self.accum_train_y.shape,
np.array(cur_data_y).shape))
else:
self.tfds_convertor_sess.close()
# return cur_incre_train_x, np.array(cur_data_y)
return {
"x": [np.squeeze(x) for x in cur_incre_train_x],
"y": np.array(cur_data_y)
}
import tensorflow as tf
import numpy as np
import time
from at_toolkit.at_utils import info, error, as_timer
from at_toolkit.interface.adl_tfds_convertor import AbsTfdsConvertor
info("note, tf version={}".format(tf.__version__))
class TfdsConvertor(AbsTfdsConvertor):
def __init__(self,
if_train_shuffle=False,
train_shuffle_size=100,
if_pad_batch=False,
padded_batch_size=20,
domain=None):
self.train_tfds = None
self.test_tfds = None
self.train_num = 0
self.test_num = 0
self.accum_train_x = list()
self.accum_train_y = None
self.accm_train_cnt = 0
self.accum_test_x = list()
self.accum_test_y = list()
self.accm_test_cnt = 0
self.tfds_train_os_iterator = None
self.tfds_train_iter_next = None
def test_pipeline(self, test_tfds):
as_timer("test_start")
# 1. raw data: tfds2np.
self.tfds_convertor.init_test_tfds(test_tfds)
if not self.feats_data_db.raw_data_db.if_raw_test_2_np_done:
raw_test_np = self.tfds_convertor.get_test_np()
assert isinstance(raw_test_np, list), "raw_test_np is not list"
info("raw_test_np, len={}, ele={}".format(len(raw_test_np), raw_test_np[0]))
self.feats_data_db.raw_data_db.put_raw_test_np(raw_test_np)
as_timer("te_s0_tfds2np_{}".format(len(raw_test_np)))
if self.cur_cls_name in [CLS_LR_LIBLINEAER, CLS_LR_SAG]:
# 2. get asso-feats test.
use_feat_params = {"len_sample": 5, "sr": 16000}
cur_test_examples_x = self.feats_data_db.get_raw_test_feats(self.cur_feat_name, use_feat_params)
as_timer("te_s1_examples_{}".format(len(cur_test_examples_x)))
# 3. cur_cls test and predict.
assert isinstance(self.cur_cls, AdlClassifier)
cur_test_preds = self.cur_cls.predict_proba(cur_test_examples_x, predict_prob_params={"if_multilabel": self.is_multilabel})
self.test_pip_id += 1
as_timer("test_end")
info(as_timer)
return np.array(cur_test_preds)
if self.cur_cls_name in [CLS_TR34]:
# tr34 train pipeline warmup.
while self.tr34_cls_train_pip_run < self.tr34_trainpip_warmup:
# while self.tr34_cls_train_pip_run < TR34_TRAINPIP_WARMUP:
self.train_pipeline(train_tfds=None, update_train_data=False)
assert isinstance(self.cur_cls, ThinResnet34Classifier), "Error, cur_cls type error."
# 2. get asso-feats test.
if_force_test_feats = self.cur_cls.decide_if_renew_testfeats()
use_feat_params = self.cur_cls.imp_feat_args
cur_test_examples_x = self.feats_data_db.get_raw_test_feats(
self.cur_feat_name, use_feat_params, if_force_test_feats
)
# need reformat for test
info("tr34_test, type={}".format(type(cur_test_examples_x)))
# cur_test_examples_x = np.array(cur_test_examples_x)
cur_test_examples_x = np.asarray(cur_test_examples_x)
cur_test_examples_x = cur_test_examples_x[:, :, :, np.newaxis]
info("tr34_test, type={}, shape={}".format(type(cur_test_examples_x), cur_test_examples_x.shape))
as_timer("te_s1_examples_{}".format(len(cur_test_examples_x)))
# 3. cur_cls test and predict.
assert isinstance(self.cur_cls, AdlClassifier)
cur_test_preds = self.cur_cls.predict_proba(cur_test_examples_x)
# del
del cur_test_examples_x
self.test_pip_id += 1
as_timer("test_end")
info(as_timer)
return cur_test_preds
as_timer("test_end")
info(as_timer)
def train_pipeline(self, train_tfds, update_train_data=True):
if self.train_pip_id < len(self.tfds2np_take_size_array):
if self.train_pip_id == 1:
take_train_size = max(200, int(self.tfds2np_take_size_array[self.train_pip_id] * self.train_num))
else:
take_train_size = int(self.tfds2np_take_size_array[self.train_pip_id] * self.train_num)
else:
take_train_size = 200
self.token_train_size += take_train_size
self.cur_train_his_report = dict()
as_timer("train_start")
# 1. raw data: tfds2np.
self.tfds_convertor.init_train_tfds(train_tfds, self.train_num)
if update_train_data is True and self.feats_data_db.raw_data_db.raw_train_np_filled_num < self.train_num:
accm_raw_train_np_dict = self.tfds_convertor.get_train_np_accm(take_train_size)
# if self.minis_eda_report is None:
self.minis_eda_report = minisamples_edaer(accm_raw_train_np_dict["x"], accm_raw_train_np_dict["y"])
# decide if re-shuffle train tfds.
if self.minis_eda_report.get("y_cover_rate") <= 0.5:
info("Warning, old_y_cover_rate={} is too low, need re_shuffle train_tfds.".format(self.minis_eda_report.get("y_cover_rate")))
self.tfds_convertor.init_train_tfds(train_tfds, self.train_num, force_shuffle=True)
# renew data, shuffle, get data and get report.
accm_raw_train_np_dict = self.tfds_convertor.get_train_np_accm(take_train_size)
self.minis_eda_report = minisamples_edaer(accm_raw_train_np_dict["x"], accm_raw_train_np_dict["y"])
info("Note, new_y_cover_rate={} ".format(self.minis_eda_report.get("y_cover_rate")))
# update: meta-is_multilabel
self.is_multilabel = self.minis_eda_report.get("is_multilabel")
self.tr34_cls.renew_if_multilabel(self.is_multilabel)
# if self.minis_eda_report is None:
if self.tfds2np_takesize_flag is False:
self.decision_maker.learn_train_minisamples_report(self.minis_eda_report)
self.tfds2np_take_size_array = self.decision_maker.decide_tfds2np_array()
self.tfds2np_takesize_flag = True
info("Note, mini_eda_report = {}, tfds2np_takesize_array={}".format(self.minis_eda_report, self.tfds2np_take_size_array))
self.feats_data_db.raw_data_db.put_raw_train_np(accm_raw_train_np_dict["x"], accm_raw_train_np_dict["y"])
# 1-1: option: split val.
if_split_val = self.decision_maker.decide_if_split_val(self.token_train_size)
info("Val: if_val_on={}, if_split_val={}, len={}".format(IF_VAL_ON, if_split_val, len(self.val_sample_idxs)))
if IF_VAL_ON and if_split_val and len(self.val_sample_idxs) == 0:
val_mode = "bal"
# val_mode = "random"
val_num = self.decision_maker.decide_g_valid_num()
self.val_sample_idxs = self.val_splitor.get_valid_sample_idxs(
np.stack(self.feats_data_db.raw_data_db.raw_train_y_np_table_filled), val_num=val_num, mode=val_mode
)
self.feats_data_db.raw_data_db.put_split_valid_np(self.val_sample_idxs)
self.cur_val_examples_y = self.feats_data_db.get_raw_train_y(self.val_sample_idxs)
info(
"Note, do val_split, mode={}, val_num={}, real_num={}, val_sampld_idxs={}".format(
val_mode, val_num, len(self.val_sample_idxs), self.val_sample_idxs
)
)
# 2. model select.
self.cur_cls_name = self.decision_maker.decide_model_select(self.train_pip_id)
info("---------Model_Select_CLS={}---------".format(self.cur_cls_name))
self.cur_cls = self.cur_cls_ins_table.get(self.cur_cls_name)
self.cur_sampler = self.cur_sampler_table.get(self.cur_cls_name)
if self.cur_cls_name in [CLS_LR_LIBLINEAER, CLS_LR_SAG]:
# 3. sample train_idxs.
# sample-even_sample.
if self.is_multilabel is False:
self.lr_sampler.init_train_y(self.feats_data_db.raw_data_db.raw_train_y_np_table_filled)
class_inverted_index_array = self.lr_sampler.init_each_class_index_by_y(self.lr_sampler.train_y)
# info("class_inverted_array={}".format(class_inverted_index_array))
info(
"class_inverted_array len={}, some={}".format(
len(class_inverted_index_array), class_inverted_index_array[:3]
)
)
cur_train_sample_idxs = self.lr_sampler.init_even_class_index_by_each(class_inverted_index_array)
info("cur_train_sample_idxs len={}, some={}".format(len(cur_train_sample_idxs), cur_train_sample_idxs[:3]))
cur_train_sample_idxs = [item for sublist in cur_train_sample_idxs for item in sublist]
info("cur_train_sample_idxs len={}, some={}".format(len(cur_train_sample_idxs), cur_train_sample_idxs[:3]))
# filter val idxs out.
cur_train_sample_idxs = [i for i in cur_train_sample_idxs if i not in self.val_sample_idxs]
info("cur_train_sample_idxs len={}, some={}".format(len(cur_train_sample_idxs), cur_train_sample_idxs[:3]))
as_timer("t_s3_trainidx_{}".format(len(cur_train_sample_idxs)))
# config: all put into: nosample
else:
cur_train_sample_idxs = range(len(self.feats_data_db.raw_data_db.raw_train_y_np_table_filled))
# 4. get asso-feats train.
self.cur_feat_name = CLS_2_FEATNAME_REG_TABLE.get(self.cur_cls_name)
self.use_feat_params = {"len_sample": 5, "sr": 16000}
cur_train_examples_x = self.feats_data_db.get_raw_train_feats(
self.cur_feat_name, cur_train_sample_idxs, self.use_feat_params
)
cur_train_examples_y = self.feats_data_db.get_raw_train_y(cur_train_sample_idxs)
# info("cur_train_examples_x, shape={}, data={}".format(cur_train_examples_x.shape, cur_train_examples_x))
# info("cur_train_examples_y, shape={}, data={}".format(cur_train_examples_y.shape, cur_train_examples_y))
info("cur_train_examples_x, shape={},".format(cur_train_examples_x.shape))
info("cur_train_examples_y, shape={},".format(cur_train_examples_y.shape))
train_eda_report = sample_y_edaer(cur_train_examples_y)
info("Note, train_eda_y_report = {}".format(train_eda_report))
as_timer("t_s4_texamples_{}".format(len(cur_train_examples_x)))
# 5. cur_cls train and fit
if self.cur_cls_name == CLS_LR_LIBLINEAER:
assert isinstance(self.cur_cls, SLLRLiblinear), "Error cur_cls is not {}".format(SLLRLiblinear.__name__)
self.cur_cls.offline_fit(cur_train_examples_x, cur_train_examples_y, fit_params={"if_multilabel": self.is_multilabel})
elif self.cur_cls_name == CLS_LR_SAG:
assert isinstance(self.cur_cls, SLLRSag), "Error cur_cls is not {}".format(SLLRSag.__name__)
self.cur_cls.offline_fit(cur_train_examples_x, cur_train_examples_y, fit_params={"if_multilabel": self.is_multilabel})
as_timer("t_s5_fit_{}".format(len(cur_train_examples_x)))
elif self.cur_cls_name in [CLS_TR34]:
assert isinstance(self.cur_cls, ThinResnet34Classifier), "Error, cls select is {}".format(
type(self.cur_cls)
)
# 3. sample train_idxs.
train_use_y_labels = np.stack(self.feats_data_db.raw_data_db.raw_train_y_np_table_filled)
info(
"train_use_y_labels, type={}, shape={}".format(
# type(train_use_y_labels), train_use_y_labels.shape, train_use_y_labels
type(train_use_y_labels), train_use_y_labels.shape
)
)
self.tr34_sampler = AutoSpSamplerNew(y_train_labels=train_use_y_labels)
if self.is_multilabel is False:
# old: use sampling.
self.tr34_sampler.set_up()
cur_train_sample_idxs = self.tr34_sampler.get_downsample_index_list_by_class(
per_class_num=Tr34SamplerHpParams.SAMPL_PA_F_PERC_NUM,
max_sample_num=self.tr34_hps_sample_dict.get("SAMP_MAX_NUM"),
min_sample_num=self.tr34_hps_sample_dict.get("SAMP_MIN_NUM"),
)
else:
# try: use all or random.
cur_train_sample_idxs = self.tr34_sampler.get_downsample_index_list_by_random(
max_sample_num=self.tr34_hps_sample_dict.get("SAMP_MAX_NUM"),
min_sample_num=self.tr34_hps_sample_dict.get("SAMP_MIN_NUM"))
info("cur_train_sample_idxs len={}, some={}".format(len(cur_train_sample_idxs), cur_train_sample_idxs[:3]))
# filter val idxs out.
cur_train_sample_idxs = [i for i in cur_train_sample_idxs if i not in self.val_sample_idxs]
info("cur_train_sample_idxs len={}, some={}".format(len(cur_train_sample_idxs), cur_train_sample_idxs[:3]))
# 4. get asso-feats train.
self.cur_feat_name = CLS_2_FEATNAME_REG_TABLE.get(self.cur_cls_name)
if_train_feats_force = self.cur_cls.decide_if_renew_trainfeats()
self.use_feat_params = self.cur_cls.imp_feat_args
info("train_feats_params={}".format(self.use_feat_params))
cur_train_examples_x = self.feats_data_db.get_raw_train_feats(
self.cur_feat_name, cur_train_sample_idxs, self.use_feat_params, if_train_feats_force
)
cur_train_examples_y = self.feats_data_db.get_raw_train_y(cur_train_sample_idxs)
# info("cur_train_examples_x, shape={}, data={}".format(cur_train_examples_x.shape, cur_train_examples_x[:1]))
info("cur_train_examples_x, shape={}".format(cur_train_examples_x.shape))
# info("cur_train_examples_y, shape={}, data={}".format(cur_train_examples_y.shape, cur_train_examples_y[:1]))
info("cur_train_examples_y, shape={}".format(cur_train_examples_y.shape))
train_eda_report = sample_y_edaer(cur_train_examples_y)
info("Note, train_eda_y_report = {}".format(train_eda_report))
as_timer("t_s4_texamples_{}".format(len(cur_train_examples_x)))
# 5. cur_cls train and fit
self.tr34_cls_train_pip_run += 1
self.cur_train_his_report = self.cur_cls.online_fit(cur_train_examples_x, cur_train_examples_y, fit_params=self.tr34_hps_epochs_dict)
info("tr34_cls_train_pip_run={}".format(self.tr34_cls_train_pip_run))
as_timer("t_s5_fit_{}".format(len(cur_train_examples_x)))
# 6. option: val
if len(self.val_sample_idxs) > 0:
# cur_val_examples_x = self.feats_data_db.get_raw_train_feats(self.cur_feat_name, self.val_sample_idxs,
# self.use_feat_params)
info("Note, cur_cls_name={}, get_val_feats done".format(self.cur_cls_name))
if self.cur_cls_name == CLS_TR34:
assert isinstance(self.cur_cls, ThinResnet34Classifier)
if_force_val_feats = self.cur_cls.decide_if_renew_valfeats()
use_feat_params = self.cur_cls.imp_feat_args
cur_val_examples_x = self.feats_data_db.get_split_val_feats(
self.cur_feat_name, self.val_sample_idxs, use_feat_params, if_force_val_feats
)
cur_val_examples_x = np.array(cur_val_examples_x)
cur_val_examples_x = cur_val_examples_x[:, :, :, np.newaxis]
cur_val_preds = self.cur_cls.predict_val_proba(cur_val_examples_x)
else:
cur_val_examples_x = self.feats_data_db.get_split_val_feats(self.cur_feat_name, self.val_sample_idxs, self.use_feat_params)
cur_val_preds = self.cur_cls.predict_proba(cur_val_examples_x, predict_prob_params={"if_multilabel": self.is_multilabel})
info("Note, cur_cls_name={}, get_val_preds done".format(self.cur_cls_name))
self.cur_val_nauc = ATEvaluator.autodl_auc(solution=self.cur_val_examples_y, prediction=cur_val_preds)
# self.cur_val_nauc = round(2*self.cur_val_auc - 1, 6)
info("Note, cur_cls_name={}, \033[1;31;m cur_val_nauc={}\033[0m".format(self.cur_cls_name, self.cur_val_nauc))
as_timer("t_s6_val_{}".format(len(self.val_sample_idxs)))
else:
# self.cur_val_auc, self.cur_val_nauc = -1, -1
self.cur_val_nauc = -1
self.train_pip_id += 1
as_timer("train_end")
info(as_timer)
self.cur_train_his_report["val_nauc"] = self.cur_val_nauc
self.cur_train_his_report["cls_name"] = self.cur_cls_name
# return self.cur_cls_name, self.cur_train_his_report
return self.cur_train_his_report.copy()