def get_lr_scheduler(**kwargs): """ return a lr_scheduler instance :param begin_epoch: the start epoch number (will work with steps) :type begin_epoch: int :param epoch_size: number of batch for each epoch (for each gpu as well) :type epoch_size: int :param steps: str like "1-2-3-4", which means decay at epoch 1,2,3,4 :type steps: str :param decay: lr * decay in each period :type decay: float :return: mx.lr_scheduler.LRScheduler """ begin_epoch = get(kwargs, 'begin_epoch', int, 0) epoch_size = get(kwargs, 'epoch_size', int, 1000) steps = get(kwargs, 'steps', lambda x: [int(i) for i in x.split('-')], [30, 50]) decay = get(kwargs, 'decay', float, 0.1) check(kwargs, 'lr_scheduler.step', ['begin_epoch', 'epoch_size', 'steps', 'decay']) step_ = [ epoch_size * (x - begin_epoch) for x in steps if x - begin_epoch > 0 ] return mx.lr_scheduler.MultiFactorScheduler( step=step_, factor=decay) if len(step_) else None
def get_monitor(**kwargs): """ return a TensorBoardMonitor instance (with custom callback) :param pattern: "a,b,c" like string, where a,b,c are regex pattern str :type pattern: str :param interval: monitor interval :type interval: int :param prefix: tensorboard name prefix :type prefix: str :return: mxnet.tensorboard.TensorBoardMonitor """ from utils.logger import get_logger pattern = get(kwargs, 'pattern', str2list, None) interval = get(kwargs, 'interval', int, 100) prefix = get(kwargs, 'prefix', str, kwargs['console_arg'].symbol_name + '.') check(kwargs, 'monitor.tensorboard', ['pattern', 'interval', 'prefix']) monitor = TensorBoardMonitor( pattern_callback_map={key: [ custom_callback, ] for key in pattern}, reject_pattern='.*label', interval=interval, profile_logging=True, logger=get_logger(), symbol_name_prefix=prefix) # tensorboard monitor per 1k batch return monitor
def get_iterator(**kwargs): """ return a tuple(pair) of iterator for cifar10 :param batch_size: e.g. 100 :type batch_size: int :param kvstore: kvstore instance :type: mx.kvstore.KVStore :param shuffle: whether to shuffle the training set :type shuffle: bool :param crop: whether to random crop the training image :type crop: bool :param mirror: whether to random mirror(flip) the training image :type mirror: bool :return: (train_iter, val_iter) :type: (mxnet.io.DataIter, mxnet.io.DataIter) """ scope_name = "iterator.cifar10" batch_size = get(kwargs, 'batch_size', int, 100) kvstore = get(kwargs, 'kvstore', lambda x: x, None) assert kvstore is not None, 'kvstore is None in ' + scope_name shuffle = get(kwargs, 'shuffle', str2bool, True) crop = get(kwargs, 'crop', str2bool, True) mirror = get(kwargs, 'mirror', str2bool, True) check(kwargs, scope_name, ['batch_size', 'shuffle', 'crop', 'mirror', 'kvstore']) kargs = dict( data_shape=(3, 28, 28), # Use mean and scale works equally well with a BatchNorm after data layer # here we use this simple method mean_r=128, mean_g=128, mean_b=128, scale=0.008, ) train = mx.io.ImageRecordIter( path_imgrec=CIFAR10['train'], batch_size=batch_size, rand_crop=crop, rand_mirror=mirror, num_parts=kvstore.num_workers, part_index=kvstore.rank, shuffle_chunk_size=10, shuffle=shuffle, **kargs ) val = mx.io.ImageRecordIter( path_imgrec=CIFAR10['val'], rand_crop=False, rand_mirror=False, batch_size=batch_size, num_parts=kvstore.num_workers, part_index=kvstore.rank, shuffle=False, **kargs ) return train, val
def get_initializer(**kwargs): """ return an Xavier initializer :param rnd_type: sampling distribution :type rnd_type: str :param factor_type: xavier fan: "in" or "out" :type factor_type: str :param magnitude: gaussian variance or uniform maximum :type magnitude: float :return: mxnet.initializer.Initializer """ result = mx.init.Xavier(rnd_type=get(kwargs, 'rnd_type', str, 'gaussian'), factor_type=get(kwargs, 'factor_type', str, 'in'), magnitude=get(kwargs, 'magnitude', float, 2.37)) check(kwargs, 'initializer.xavier', ['rnd_type', 'factor_type', 'magnitude']) return result
def get_session(): gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=1.0, visible_device_list=args.get('gpu', '0'), allow_growth=True, ) config = tf.ConfigProto(gpu_options=gpu_options) session = tf.Session(config=config) return session
def get_optimizer(**kwargs): """ return a legacyadam optimizer, it works like https://stackoverflow.com/questions/44452571/what-is-the-proper-way-to-weight-decay-for-adam-optimizer :param lr: learning rate :param lr: float :param wd: weight decay factor :type wd: float :param clip: gradient clip :type clip: Union[None, float] :return: mxnet.optimizer.Optimizer """ result = { "optimizer": "legacyadam", "optimizer_params": { 'learning_rate': get(kwargs, "lr", float, 0.1), 'wd': get(kwargs, "wd", float, 5e-4), 'clip_gradient': get(kwargs, "clip", float, None) } } check(kwargs, 'optimizer.legacyadam', ['lr', 'wd', 'clip']) return result
def l2_loss(name): alpha = args.get(f'l2_{name}', 0) if alpha < 1e-10: return None return lambda x: alpha * tf.nn.l2_loss(x)
def __init__(self): self.home = f'{data_home}/{args.ds}' metadata = self.load_json('metadata') args.update(**metadata) tmp = [6, 11, 15, 18, 22, 24, 26, 29, 32, 33] if args.ds == 'v8': tmp = tmp[:7] global NUM_PHASE NUM_PHASE = 7 # tmp = [t + 2 for t in tmp] metadata['mid_deg_per_phase'] = tmp self.mid_deg_per_phase = metadata['mid_deg_per_phase'] args.update(mid_deg_per_phase=str(self.mid_deg_per_phase)) self.user2item_seq = self.load_json('user2item_seq') self.user2ts_seq = self.load_json('user2ts_seq') self.vali_puiqa = self.load_json('vali_puiqa') self.test_puiqa = self.load_json('test_puiqa') # uids_list, vids_list self.raw_id_list = self.load_json('ids_list') try: self.item_feat = np.load(f'{self.home}/item_feat.npy') except Exception as e: print(e) self.item_feat = np.zeros([args.nb_items, 2, 128]) self.item_deg_per_phase = self.load_json('item_deg') self.item_deg_per_phase = np.array(self.item_deg_per_phase, dtype=int) self.item_deg_self_per_phase = self.load_json('item_deg_self') self.item_deg_self_per_phase = np.array(self.item_deg_self_per_phase, dtype=int) self.item_is_half = self.item_deg_per_phase <= np.array( self.mid_deg_per_phase)[:, None] self.score_mask_per_phase = np.ones([NUM_PHASE, args.nb_items], dtype=int) self.score_mask_per_phase[self.item_deg_self_per_phase == 0] = 0 self.vali_user2ppa = {} for phase, user, pos, ts, ans in self.vali_puiqa: self.vali_user2ppa[user] = (phase, pos, ans) self.test_user2ppa = {} for phase, user, pos, ts, ans in self.test_puiqa: self.test_user2ppa[user] = (phase, pos, ans) self.train_users = list(range(args.nb_users)) self.vali_users = sorted(self.vali_user2ppa.keys()) self.test_users = sorted(self.test_user2ppa.keys()) self.item_pop = [0] * args.nb_items self.user_pop = [0] * args.nb_users self.for_phase = -1 if len(args.mode_pred_phase) == 1: self.for_phase = int(args.mode_pred_phase) args.update(for_phase=self.for_phase) nb_train = 0 for user, item_list in enumerate(self.user2item_seq): for i, item in enumerate(item_list): if item == 1: vali_phase, vali_pos, vali_ans = self.vali_user2ppa[user] if args.use_unused_vali and args.mode_pred_phase != 'all' and str( vali_phase) not in args.mode_pred_phase: item = vali_ans if item >= 3: self.item_pop[item] += 1 self.user_pop[user] += 1 nb_train += 1 if args.mode_resample != 'none': if args.mode_resample == 'user': if user in self.vali_user2ppa or user in self.test_user2ppa: nb_train += args.alpha_resample elif args.mode_resample == 'rare': # assert len(args.mode_pred_phase) == 1 # assert phase == int(args.mode_pred_phase) phase = int(args.mode_pred_phase) item_deg = self.item_deg_per_phase[phase][item] mid_deg = self.mid_deg_per_phase[phase] if item_deg <= mid_deg: nb_train += args.alpha_resample elif args.mode_resample == 'phase': phase = int(args.mode_pred_phase) q_ts = self.user2ts_seq[user][i] unit = 54.5 min_ts = phase * unit max_ts = min_ts + (unit * 4) + 1 if min_ts <= q_ts < max_ts: nb_train += args.alpha_resample elif args.mode_resample == 'day': phase = int(args.mode_pred_phase) q_ts = self.user2ts_seq[user][i] unit = 54.5 min_ts = 10.0 + (phase + 3) * unit max_ts = min_ts + unit + 1 if min_ts <= q_ts < max_ts: nb_train += args.alpha_resample for item in range(3): self.item_pop[item] += 1 self.item_pop = np.array(self.item_pop, dtype=int) self.item_pop[self.item_pop == 0] = 1 assert np.min(self.item_pop) >= 1 self.item_pop_log = np.log(self.item_pop + np.e) self.item_pop_inv = 1.0 / self.item_pop self.item_pop_log_inv = 1.0 / self.item_pop_log self.user_pop = np.array(self.user_pop, dtype=int) args.update(nb_train=nb_train) if args.gnn: pass if args.get('run_data', False) and args.run_test: pass
def fuse(ans_list, weight_list, name): users = sorted(ans_list[0].keys()) user2tops = {} global max_score_per_ans, score_diff_per_ans if max_score_per_ans is None: max_score_per_ans = [] score_diff_per_ans = [] for i in range(len(ans_list)): max_score, min_score = None, None for user in tqdm(users, desc='count min max', ncols=90, ascii=True): logits_dict = ans_list[i][user] for item, score in logits_dict.items(): if max_score is None: max_score = score min_score = score else: max_score = max(max_score, score) min_score = min(min_score, score) print(f'ans {i}, min ~ max: {min_score} ~ {max_score}') score_diff = max_score - min_score max_score_per_ans.append(max_score) score_diff_per_ans.append(score_diff) if name == 'vali': user2ppa = data.dp.vali_user2ppa else: user2ppa = data.dp.test_user2ppa is_first_fuse = args.get('is_first_fuse', True) for user in tqdm(users, desc='fuse', ncols=90, ascii=True, leave=is_first_fuse): phase, pos, ans = user2ppa[user] final_ans = defaultdict(float) for ans_i, (ans, weight) in enumerate(zip(ans_list, weight_list)): if weight < 1e-6: continue logits_dict = ans[user] for item, score in logits_dict.items(): pop_inv = data.dp.item_pop_inv[item] pop_log_inv = data.dp.item_pop_log_inv[item] s = score / score_diff_per_ans[ans_i] if args.mode_pop == 'log': s *= (1.0 + pop_log_inv) if args.mode_rare == 'linear': if data.dp.item_is_half[phase, item] and alpha_rare[ans_i] > 0: s *= (alpha_rare[ans_i] + pop_inv) w = weight final_ans[item] += w * s tops = sorted(final_ans.keys(), key=lambda _item: (-final_ans[_item], _item))[:args.nb_topk] user2tops[user] = tops args.update(is_first_fuse=False) args.update(show_detail=False) return user2tops