def get_lr_scheduler(**kwargs):
"""
return a lr_scheduler instance
:param begin_epoch: the start epoch number (will work with steps)
:type begin_epoch: int
:param epoch_size: number of batch for each epoch (for each gpu as well)
:type epoch_size: int
:param steps: str like "1-2-3-4", which means decay at epoch 1,2,3,4
:type steps: str
:param decay: lr * decay in each period
:type decay: float
:return: mx.lr_scheduler.LRScheduler
"""
begin_epoch = get(kwargs, 'begin_epoch', int, 0)
epoch_size = get(kwargs, 'epoch_size', int, 1000)
steps = get(kwargs, 'steps', lambda x: [int(i) for i in x.split('-')],
[30, 50])
decay = get(kwargs, 'decay', float, 0.1)
check(kwargs, 'lr_scheduler.step',
['begin_epoch', 'epoch_size', 'steps', 'decay'])
step_ = [
epoch_size * (x - begin_epoch) for x in steps if x - begin_epoch > 0
]
return mx.lr_scheduler.MultiFactorScheduler(
step=step_, factor=decay) if len(step_) else None
def get_monitor(**kwargs):
"""
return a TensorBoardMonitor instance (with custom callback)
:param pattern: "a,b,c" like string, where a,b,c are regex pattern str
:type pattern: str
:param interval: monitor interval
:type interval: int
:param prefix: tensorboard name prefix
:type prefix: str
:return: mxnet.tensorboard.TensorBoardMonitor
"""
from utils.logger import get_logger
pattern = get(kwargs, 'pattern', str2list, None)
interval = get(kwargs, 'interval', int, 100)
prefix = get(kwargs, 'prefix', str,
kwargs['console_arg'].symbol_name + '.')
check(kwargs, 'monitor.tensorboard', ['pattern', 'interval', 'prefix'])
monitor = TensorBoardMonitor(
pattern_callback_map={key: [
custom_callback,
]
for key in pattern},
reject_pattern='.*label',
interval=interval,
profile_logging=True,
logger=get_logger(),
symbol_name_prefix=prefix) # tensorboard monitor per 1k batch
return monitor
def get_iterator(**kwargs):
"""
return a tuple(pair) of iterator for cifar10
:param batch_size: e.g. 100
:type batch_size: int
:param kvstore: kvstore instance
:type: mx.kvstore.KVStore
:param shuffle: whether to shuffle the training set
:type shuffle: bool
:param crop: whether to random crop the training image
:type crop: bool
:param mirror: whether to random mirror(flip) the training image
:type mirror: bool
:return: (train_iter, val_iter)
:type: (mxnet.io.DataIter, mxnet.io.DataIter)
"""
scope_name = "iterator.cifar10"
batch_size = get(kwargs, 'batch_size', int, 100)
kvstore = get(kwargs, 'kvstore', lambda x: x, None)
assert kvstore is not None, 'kvstore is None in ' + scope_name
shuffle = get(kwargs, 'shuffle', str2bool, True)
crop = get(kwargs, 'crop', str2bool, True)
mirror = get(kwargs, 'mirror', str2bool, True)
check(kwargs, scope_name, ['batch_size', 'shuffle', 'crop', 'mirror', 'kvstore'])
kargs = dict(
data_shape=(3, 28, 28),
# Use mean and scale works equally well with a BatchNorm after data layer
# here we use this simple method
mean_r=128,
mean_g=128,
mean_b=128,
scale=0.008,
)
train = mx.io.ImageRecordIter(
path_imgrec=CIFAR10['train'],
batch_size=batch_size,
rand_crop=crop,
rand_mirror=mirror,
num_parts=kvstore.num_workers,
part_index=kvstore.rank,
shuffle_chunk_size=10,
shuffle=shuffle,
**kargs
)
val = mx.io.ImageRecordIter(
path_imgrec=CIFAR10['val'],
rand_crop=False,
rand_mirror=False,
batch_size=batch_size,
num_parts=kvstore.num_workers,
part_index=kvstore.rank,
shuffle=False,
**kargs
)
return train, val
def get_initializer(**kwargs):
"""
return an Xavier initializer
:param rnd_type: sampling distribution
:type rnd_type: str
:param factor_type: xavier fan: "in" or "out"
:type factor_type: str
:param magnitude: gaussian variance or uniform maximum
:type magnitude: float
:return: mxnet.initializer.Initializer
"""
result = mx.init.Xavier(rnd_type=get(kwargs, 'rnd_type', str, 'gaussian'),
factor_type=get(kwargs, 'factor_type', str, 'in'),
magnitude=get(kwargs, 'magnitude', float, 2.37))
check(kwargs, 'initializer.xavier',
['rnd_type', 'factor_type', 'magnitude'])
return result
def get_session():
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=1.0,
visible_device_list=args.get('gpu', '0'),
allow_growth=True,
)
config = tf.ConfigProto(gpu_options=gpu_options)
session = tf.Session(config=config)
return session
def get_optimizer(**kwargs):
"""
return a legacyadam optimizer, it works like
https://stackoverflow.com/questions/44452571/what-is-the-proper-way-to-weight-decay-for-adam-optimizer
:param lr: learning rate
:param lr: float
:param wd: weight decay factor
:type wd: float
:param clip: gradient clip
:type clip: Union[None, float]
:return: mxnet.optimizer.Optimizer
"""
result = {
"optimizer": "legacyadam",
"optimizer_params": {
'learning_rate': get(kwargs, "lr", float, 0.1),
'wd': get(kwargs, "wd", float, 5e-4),
'clip_gradient': get(kwargs, "clip", float, None)
}
}
check(kwargs, 'optimizer.legacyadam', ['lr', 'wd', 'clip'])
return result
def l2_loss(name):
alpha = args.get(f'l2_{name}', 0)
if alpha < 1e-10:
return None
return lambda x: alpha * tf.nn.l2_loss(x)
def __init__(self):
self.home = f'{data_home}/{args.ds}'
metadata = self.load_json('metadata')
args.update(**metadata)
tmp = [6, 11, 15, 18, 22, 24, 26, 29, 32, 33]
if args.ds == 'v8':
tmp = tmp[:7]
global NUM_PHASE
NUM_PHASE = 7
# tmp = [t + 2 for t in tmp]
metadata['mid_deg_per_phase'] = tmp
self.mid_deg_per_phase = metadata['mid_deg_per_phase']
args.update(mid_deg_per_phase=str(self.mid_deg_per_phase))
self.user2item_seq = self.load_json('user2item_seq')
self.user2ts_seq = self.load_json('user2ts_seq')
self.vali_puiqa = self.load_json('vali_puiqa')
self.test_puiqa = self.load_json('test_puiqa')
# uids_list, vids_list
self.raw_id_list = self.load_json('ids_list')
try:
self.item_feat = np.load(f'{self.home}/item_feat.npy')
except Exception as e:
print(e)
self.item_feat = np.zeros([args.nb_items, 2, 128])
self.item_deg_per_phase = self.load_json('item_deg')
self.item_deg_per_phase = np.array(self.item_deg_per_phase, dtype=int)
self.item_deg_self_per_phase = self.load_json('item_deg_self')
self.item_deg_self_per_phase = np.array(self.item_deg_self_per_phase,
dtype=int)
self.item_is_half = self.item_deg_per_phase <= np.array(
self.mid_deg_per_phase)[:, None]
self.score_mask_per_phase = np.ones([NUM_PHASE, args.nb_items],
dtype=int)
self.score_mask_per_phase[self.item_deg_self_per_phase == 0] = 0
self.vali_user2ppa = {}
for phase, user, pos, ts, ans in self.vali_puiqa:
self.vali_user2ppa[user] = (phase, pos, ans)
self.test_user2ppa = {}
for phase, user, pos, ts, ans in self.test_puiqa:
self.test_user2ppa[user] = (phase, pos, ans)
self.train_users = list(range(args.nb_users))
self.vali_users = sorted(self.vali_user2ppa.keys())
self.test_users = sorted(self.test_user2ppa.keys())
self.item_pop = [0] * args.nb_items
self.user_pop = [0] * args.nb_users
self.for_phase = -1
if len(args.mode_pred_phase) == 1:
self.for_phase = int(args.mode_pred_phase)
args.update(for_phase=self.for_phase)
nb_train = 0
for user, item_list in enumerate(self.user2item_seq):
for i, item in enumerate(item_list):
if item == 1:
vali_phase, vali_pos, vali_ans = self.vali_user2ppa[user]
if args.use_unused_vali and args.mode_pred_phase != 'all' and str(
vali_phase) not in args.mode_pred_phase:
item = vali_ans
if item >= 3:
self.item_pop[item] += 1
self.user_pop[user] += 1
nb_train += 1
if args.mode_resample != 'none':
if args.mode_resample == 'user':
if user in self.vali_user2ppa or user in self.test_user2ppa:
nb_train += args.alpha_resample
elif args.mode_resample == 'rare':
# assert len(args.mode_pred_phase) == 1
# assert phase == int(args.mode_pred_phase)
phase = int(args.mode_pred_phase)
item_deg = self.item_deg_per_phase[phase][item]
mid_deg = self.mid_deg_per_phase[phase]
if item_deg <= mid_deg:
nb_train += args.alpha_resample
elif args.mode_resample == 'phase':
phase = int(args.mode_pred_phase)
q_ts = self.user2ts_seq[user][i]
unit = 54.5
min_ts = phase * unit
max_ts = min_ts + (unit * 4) + 1
if min_ts <= q_ts < max_ts:
nb_train += args.alpha_resample
elif args.mode_resample == 'day':
phase = int(args.mode_pred_phase)
q_ts = self.user2ts_seq[user][i]
unit = 54.5
min_ts = 10.0 + (phase + 3) * unit
max_ts = min_ts + unit + 1
if min_ts <= q_ts < max_ts:
nb_train += args.alpha_resample
for item in range(3):
self.item_pop[item] += 1
self.item_pop = np.array(self.item_pop, dtype=int)
self.item_pop[self.item_pop == 0] = 1
assert np.min(self.item_pop) >= 1
self.item_pop_log = np.log(self.item_pop + np.e)
self.item_pop_inv = 1.0 / self.item_pop
self.item_pop_log_inv = 1.0 / self.item_pop_log
self.user_pop = np.array(self.user_pop, dtype=int)
args.update(nb_train=nb_train)
if args.gnn:
pass
if args.get('run_data', False) and args.run_test:
pass
def fuse(ans_list, weight_list, name):
users = sorted(ans_list[0].keys())
user2tops = {}
global max_score_per_ans, score_diff_per_ans
if max_score_per_ans is None:
max_score_per_ans = []
score_diff_per_ans = []
for i in range(len(ans_list)):
max_score, min_score = None, None
for user in tqdm(users, desc='count min max', ncols=90,
ascii=True):
logits_dict = ans_list[i][user]
for item, score in logits_dict.items():
if max_score is None:
max_score = score
min_score = score
else:
max_score = max(max_score, score)
min_score = min(min_score, score)
print(f'ans {i}, min ~ max: {min_score} ~ {max_score}')
score_diff = max_score - min_score
max_score_per_ans.append(max_score)
score_diff_per_ans.append(score_diff)
if name == 'vali':
user2ppa = data.dp.vali_user2ppa
else:
user2ppa = data.dp.test_user2ppa
is_first_fuse = args.get('is_first_fuse', True)
for user in tqdm(users,
desc='fuse',
ncols=90,
ascii=True,
leave=is_first_fuse):
phase, pos, ans = user2ppa[user]
final_ans = defaultdict(float)
for ans_i, (ans, weight) in enumerate(zip(ans_list, weight_list)):
if weight < 1e-6:
continue
logits_dict = ans[user]
for item, score in logits_dict.items():
pop_inv = data.dp.item_pop_inv[item]
pop_log_inv = data.dp.item_pop_log_inv[item]
s = score / score_diff_per_ans[ans_i]
if args.mode_pop == 'log':
s *= (1.0 + pop_log_inv)
if args.mode_rare == 'linear':
if data.dp.item_is_half[phase,
item] and alpha_rare[ans_i] > 0:
s *= (alpha_rare[ans_i] + pop_inv)
w = weight
final_ans[item] += w * s
tops = sorted(final_ans.keys(),
key=lambda _item:
(-final_ans[_item], _item))[:args.nb_topk]
user2tops[user] = tops
args.update(is_first_fuse=False)
args.update(show_detail=False)
return user2tops
