def build_model(self): # some parameters image_dims = [self.input_height, self.input_width, self.c_dim] bs = self.batch_size """ Graph Input """ # images self.inputs = tf.placeholder(tf.float32, [bs] + image_dims, name='real_images') # noises self.z = tf.placeholder(tf.float32, [bs, self.z_dim], name='z') """ Loss Function """ # output of D for real images D_real, D_real_logits, _ = self.discriminator(self.inputs, is_training=True, reuse=False) # output of D for fake images G = self.generator(self.z, is_training=True, reuse=False) D_fake, D_fake_logits, _ = self.discriminator(G, is_training=True, reuse=True) # get loss for discriminator d_loss_real = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=D_real_logits, labels=tf.ones_like(D_real))) d_loss_fake = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=D_fake_logits, labels=tf.zeros_like(D_fake))) self.d_loss = d_loss_real + d_loss_fake # get loss for generator self.g_loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=D_fake_logits, labels=tf.ones_like(D_fake))) """ Training """ # divide trainable variables into a group for D and a group for G t_vars = tf.trainable_variables() d_vars = [var for var in t_vars if 'd_' in var.name] g_vars = [var for var in t_vars if 'g_' in var.name] # optimizers with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)): self.d_optim = YFOptimizer(self.learning_rate) \ .minimize(self.d_loss, var_list=d_vars) self.g_optim = YFOptimizer(self.learning_rate*5) \ .minimize(self.g_loss, var_list=g_vars) """" Testing """ # for test self.fake_images = self.generator(self.z, is_training=False, reuse=True) """ Summary """ d_loss_real_sum = tf.summary.scalar("d_loss_real", d_loss_real) d_loss_fake_sum = tf.summary.scalar("d_loss_fake", d_loss_fake) d_loss_sum = tf.summary.scalar("d_loss", self.d_loss) g_loss_sum = tf.summary.scalar("g_loss", self.g_loss) # final summary operations self.g_sum = tf.summary.merge([d_loss_fake_sum, g_loss_sum]) self.d_sum = tf.summary.merge([d_loss_real_sum, d_loss_sum])
def _configure_optimizer(learning_rate): """Configures the optimizer used for training. Args: learning_rate: A scalar or `Tensor` learning rate. Returns: An instance of an optimizer. Raises: ValueError: if FLAGS.optimizer is not recognized. """ if FLAGS.optimizer == 'adadelta': optimizer = tf.train.AdadeltaOptimizer(learning_rate, rho=FLAGS.adadelta_rho, epsilon=FLAGS.opt_epsilon) elif FLAGS.optimizer == 'adagrad': optimizer = tf.train.AdagradOptimizer( learning_rate, initial_accumulator_value=FLAGS.adagrad_initial_accumulator_value) elif FLAGS.optimizer == 'adam': optimizer = tf.train.AdamOptimizer(learning_rate, beta1=FLAGS.adam_beta1, beta2=FLAGS.adam_beta2, epsilon=FLAGS.opt_epsilon) elif FLAGS.optimizer == 'ftrl': optimizer = tf.train.FtrlOptimizer( learning_rate, learning_rate_power=FLAGS.ftrl_learning_rate_power, initial_accumulator_value=FLAGS.ftrl_initial_accumulator_value, l1_regularization_strength=FLAGS.ftrl_l1, l2_regularization_strength=FLAGS.ftrl_l2) elif FLAGS.optimizer == 'momentum': optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum, name='Momentum') elif FLAGS.optimizer == 'rmsprop': optimizer = tf.train.RMSPropOptimizer(learning_rate, decay=FLAGS.rmsprop_decay, momentum=FLAGS.rmsprop_momentum, epsilon=FLAGS.opt_epsilon) elif FLAGS.optimizer == 'sgd': optimizer = tf.train.GradientDescentOptimizer(learning_rate) elif FLAGS.optimizer == 'yellowfin': from yellowfin import YFOptimizer if learning_rate is None: optimizer = YFOptimizer() else: assert type(learning_rate ) is float, 'yellowfin uses a numerical learning_rate' optimizer = YFOptimizer(learning_rate=learning_rate, momentum=FLAGS.momentum) else: raise ValueError('Optimizer [%s] was not recognized', FLAGS.optimizer) return optimizer
def _optimizer(self, global_step): # 选择优化器 if self.optimizer_type == "adam": optimizer = tf.compat.v1.train.AdamOptimizer( learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss, global_step=global_step) elif self.optimizer_type == "adadelta": optimizer = tf.compat.v1.train.AdadeltaOptimizer( learning_rate=self.learning_rate).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "adagrad": optimizer = tf.compat.v1.train.AdagradOptimizer( learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "gd" or self.optimizer_type == "sgd": optimizer = tf.compat.v1.train.GradientDescentOptimizer( learning_rate=self.learning_rate).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "ftrl": optimizer = tf.compat.v1.train.FtrlOptimizer( learning_rate=self.learning_rate).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "padagrad": optimizer = tf.compat.v1.train.ProximalAdagradOptimizer( learning_rate=self.learning_rate).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "pgd": optimizer = tf.compat.v1.train.ProximalGradientDescentOptimizer( learning_rate=self.learning_rate).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "rmsprop": optimizer = tf.compat.v1.train.RMSPropOptimizer( learning_rate=self.learning_rate).minimize( self.loss, global_step=global_step) elif self.optimizer_type == "momentum": optimizer = tf.compat.v1.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.2).minimize(self.loss, global_step=global_step) elif self.optimizer_type == "yellowfin": optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize( self.loss, global_step=global_step) else: raise ValueError("this optimizer is undefined{0}".format( self.optimizer_type)) return optimizer
def get_optimizer(z, learning_rate, hparams): if hparams.optimizer_type == 'sgd': return torch.optim.SGD([z], learning_rate, momentum=hparams.momentum) elif hparams.optimizer_type == 'yellowfin': return YFOptimizer([z], learning_rate) elif hparams.optimizer_type == 'rmsprop': return torch.optim.RMSprop([z], lr=learning_rate, momentum=hparams.momentum) elif hparams.optimizer_type == 'adam': return torch.optim.Adam([z], lr=learning_rate) elif hparams.optimizer_type == 'adagrad': return torch.optim.Adagrad([z], lr=learning_rate) else: raise Exception('Optimizer ' + hparams.optimizer_type + ' not supported')
def main(): global args, best_loss args = parser.parse_args() configure("runs/%s" % (args.name)) model = MarketLSTM(args.layers) # get the number of model parameters print('Number of model parameters: {}'.format( sum([p.data.nelement() for p in model.parameters()]))) kwargs = {'num_workers': 1, 'pin_memory': True} model = model.cuda().double() cudnn.benchmark = True # define loss function (criterion) and pptimizer criterion = nn.MSELoss().cuda() if args.adam: optimizer = torch.optim.Adam(model.parameters(), args.lr, weight_decay=args.weight_decay) elif args.yf: optimizer = YFOptimizer(model.parameters(), lr=args.lr) else: optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) for epoch in range(epochs): train(train_loader(), model, criterion, optimizer, epoch) loss = validate(val_loader(), model, criterion, epoch) # remember best prec@1 and save checkpoint is_best = loss < best_loss best_loss = min(loss, best_loss) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec': best_loss, }, is_best) print('Best accuracy: ', best_loss)
def test_measurement(): opt = YFOptimizer(zero_debias=False) w = tf.Variable(np.ones([n_dim, ] ), dtype=tf.float32, name="w", trainable=True) b = tf.Variable(np.ones([1, ], dtype=np.float32), dtype=tf.float32, name="b", trainable=True) x = tf.constant(np.ones([n_dim, ], dtype=np.float32), dtype=tf.float32) loss = tf.multiply(w, x) + b tvars = tf.trainable_variables() w_grad_val = tf.placeholder(tf.float32, shape=(n_dim, ) ) b_grad_val = tf.placeholder(tf.float32, shape=(1, ) ) apply_op = opt.apply_gradients(zip([w_grad_val, b_grad_val], tvars) ) init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) target_h_max = 0.0 target_h_min = 0.0 g_norm_squared_avg = 0.0 g_norm_avg = 0.0 g_avg = 0.0 target_dist = 0.0 for i in range(n_iter): feed_dict = {w_grad_val: (i + 1) * np.ones( [n_dim, ], dtype=np.float32), b_grad_val: (i + 1) * np.ones( [1, ], dtype=np.float32) } res = sess.run( [opt._curv_win, opt._h_max, opt._h_min, opt._grad_var, opt._dist_to_opt_avg, apply_op], feed_dict=feed_dict) g_norm_squared_avg = 0.999 * g_norm_squared_avg \ + 0.001 * np.sum(( (i + 1)*np.ones( [n_dim + 1, ] ) )**2) g_norm_avg = 0.999 * g_norm_avg \ + 0.001 * np.linalg.norm( (i + 1)*np.ones( [n_dim + 1, ] ) ) g_avg = 0.999 * g_avg + 0.001 * (i + 1) target_h_max = 0.999 * target_h_max + 0.001 * (i + 1)**2*(n_dim + 1) target_h_min = 0.999 * target_h_min + 0.001 * max(1, i + 2 - 20)**2*(n_dim + 1) target_var = g_norm_squared_avg - g_avg**2 * (n_dim + 1) target_dist = 0.999 * target_dist + 0.001 * g_norm_avg / g_norm_squared_avg # print "iter ", i, " h max ", res[1], target_h_max, " h min ", res[2], target_h_min, \ # " var ", res[3], target_var, " dist ", res[4], target_dist assert np.abs(target_h_max - res[1] ) < np.abs(target_h_max) * 1e-3 assert np.abs(target_h_min - res[2] ) < np.abs(target_h_min) * 1e-3 assert np.abs(target_var - res[3] ) < np.abs(res[3] ) * 1e-3 assert np.abs(target_dist - res[4] ) < np.abs(res[4] ) * 1e-3 print "sync measurement test passed!"
def _init_graph(self): self.graph = tf.Graph() with self.graph.as_default(): tf.set_random_seed(self.random_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") # None * F self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") # None * F self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1 self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm") self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") self.weights = self._initialize_weights() # model self.embeddings = tf.nn.embedding_lookup( self.weights["feature_embeddings"], self.feat_index) # None * F * K feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1]) self.embeddings = tf.multiply(self.embeddings, feat_value) # ---------- first order term ---------- self.y_first_order = tf.nn.embedding_lookup( self.weights["feature_bias"], self.feat_index) # None * F * 1 self.y_first_order = tf.reduce_sum( tf.multiply(self.y_first_order, feat_value), 2) # None * F self.y_first_order = tf.nn.dropout( self.y_first_order, self.dropout_keep_fm[0]) # None * F # ---------- second order term --------------- # sum_square part self.summed_features_emb = tf.reduce_sum(self.embeddings, 1) # None * K self.summed_features_emb_square = tf.square( self.summed_features_emb) # None * K # square_sum part self.squared_features_emb = tf.square(self.embeddings) self.squared_sum_features_emb = tf.reduce_sum( self.squared_features_emb, 1) # None * K # second order self.y_second_order = 0.5 * tf.subtract( self.summed_features_emb_square, self.squared_sum_features_emb) # None * K self.y_second_order = tf.nn.dropout( self.y_second_order, self.dropout_keep_fm[1]) # None * K # high order if self.use_fm and self.use_deep: z = tf.layers.Dense( self.embedding_size, kernel_initializer=tf.glorot_uniform_initializer( seed=2017), dtype=tf.float32, bias_initializer=tf.zeros_initializer())( self.y_second_order) z = tf.nn.relu(z) y_second_order = tf.nn.dropout(z, 0.5) if self.use_xfm: field_nums = [self.field_size] final_len = 0 # self.embeddings = None * F * K hidden_nn_layers = [self.embeddings] final_result = [] split_tensor0 = tf.split(hidden_nn_layers[-1], self.embedding_size * [1], 2) for idx, layer_size in enumerate([self.field_size] * 3): split_tensor = tf.split(hidden_nn_layers[-1], self.embedding_size * [1], 2) dot_result_m = tf.matmul(split_tensor0, split_tensor, transpose_b=True) dot_result_o = tf.reshape( dot_result_m, shape=[ self.embedding_size, -1, field_nums[0] * field_nums[-1] ]) dot_result = tf.transpose(dot_result_o, perm=[1, 0, 2]) filters = tf.get_variable( name="f_" + str(idx), shape=[1, field_nums[-1] * field_nums[0], layer_size], dtype=tf.float32) curr_out = tf.nn.conv1d(dot_result, filters=filters, stride=1, padding='VALID') # if bians: b = tf.get_variable(name="f_b" + str(idx), shape=[layer_size], dtype=tf.float32, initializer=tf.zeros_initializer()) curr_out = tf.nn.bias_add(curr_out, b) curr_out = tf.nn.relu(curr_out) curr_out = tf.transpose(curr_out, perm=[0, 2, 1]) direct_connect = curr_out next_hidden = curr_out final_len += layer_size field_nums.append(int(layer_size)) final_result.append(direct_connect) hidden_nn_layers.append(next_hidden) result = tf.concat(final_result, axis=1) result = tf.reduce_sum(result, -1) # res net w_nn_output1 = tf.get_variable(name='w_nn_output1', shape=[final_len, 128], dtype=tf.float32) b_nn_output1 = tf.get_variable( name='b_nn_output1', shape=[128], dtype=tf.float32, initializer=tf.zeros_initializer()) exFM_out0 = tf.nn.xw_plus_b(result, w_nn_output1, b_nn_output1) exFM_out1 = tf.nn.dropout(exFM_out0, 0.3) exFM_out1 = tf.nn.relu(exFM_out1) w_nn_output2 = tf.get_variable( name='w_nn_output2', shape=[128 + final_len, self.embedding_size], dtype=tf.float32) b_nn_output2 = tf.get_variable( name='b_nn_output2', shape=[self.embedding_size], dtype=tf.float32, initializer=tf.zeros_initializer()) exFM_in = tf.concat([exFM_out1, result], axis=1, name="user_emb") self.exFM_out = tf.nn.xw_plus_b(exFM_in, w_nn_output2, b_nn_output2) # ---------- Deep component ---------- self.y_deep = tf.reshape( self.embeddings, shape=[-1, self.field_size * self.embedding_size]) # None * (F*K) self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0]) for i in range(0, len(self.deep_layers)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.weights["layer_%d" % i]), self.weights["bias_%d" % i]) # None * layer[i] * 1 if self.batch_norm: self.y_deep = self.batch_norm_layer( self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" % i) # None * layer[i] * 1 self.y_deep = self.deep_layers_activation(self.y_deep) self.y_deep = tf.nn.dropout( self.y_deep, self.dropout_keep_deep[1 + i]) # dropout at each Deep layer # ---------- DeepFM ---------- if self.use_fm and self.use_deep: concat_input = tf.concat( [self.y_first_order, self.y_second_order, self.y_deep], axis=1) elif self.use_xfm and self.use_deep: concat_input = tf.concat( [self.y_first_order, self.exFM_out, self.y_deep], axis=1) elif self.use_fm: concat_input = tf.concat( [self.y_first_order, self.y_second_order], axis=1) elif self.use_deep: concat_input = self.y_deep self.out = tf.add( tf.matmul(concat_input, self.weights["concat_projection"]), self.weights["concat_bias"]) # loss if self.loss_type == "logloss": self.out = tf.nn.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == "mse": self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # l2 regularization on weights if self.l2_reg > 0: self.loss += tf.contrib.layers.l2_regularizer(self.l2_reg)( self.weights["concat_projection"]) if self.use_deep: for i in range(len(self.deep_layers)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["layer_%d" % i]) # optimizer if self.optimizer_type == "adam": self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss) elif self.optimizer_type == "adagrad": self.optimizer = tf.train.AdagradOptimizer( learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize(self.loss) elif self.optimizer_type == "gd": self.optimizer = tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate).minimize(self.loss) elif self.optimizer_type == "momentum": self.optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.95).minimize(self.loss) elif self.optimizer_type == "yellowfin": self.optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize(self.loss) # init self.model_path = './model' self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = self._init_session() self.sess.run(init) # number of params total_parameters = 0 for variable in self.weights.values(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters if self.verbose > 0: print("#params: %d" % total_parameters)
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) # Data if 'cifar' in args.dataset: print('==> Preparing cifar dataset %s' % args.dataset) transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 else: dataloader = datasets.CIFAR100 num_classes = 100 trainset = dataloader(root=os.path.join(homeDir,args.dataset), train=True, download=True, transform=transform_train) train_sampler=None toShuffle = True if args.subsample < 1: toShuffle=False n = int(float(len(trainset)) * args.subsample) assert n > 0,'must sample a positive number of training examples.' train_sampler = data.sampler.SubsetRandomSampler(range(n)) print('==>SAMPLING FIRST',n,'TRAINING IMAGES') if args.class_subset != '_': print('*'+args.class_subset+'*') toShuffle = False args.class_subset = [int(i) for i in args.class_subset.split('_')] indices = [i for i, p in enumerate(trainset.train_labels) if p in args.class_subset] train_sampler = data.sampler.SubsetRandomSampler(indices); trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=toShuffle, num_workers=args.workers,sampler=train_sampler) testset = dataloader(root=os.path.join(homeDir ,args.dataset), train=False, download=False, transform=transform_test) test_sampler=None if args.test_subsample < 1: n = int(float(len(testset)) * args.test_subsample) assert n > 0, 'must sample a positive number of training examples.' test_sampler = data.sampler.SubsetRandomSampler(range(n)) print('==>SAMPLING FIRST', n, 'TESTTING IMAGES') if type(args.class_subset) is list: toShuffle = False indices = [i for i, p in enumerate(testset.test_labels) if p in args.class_subset] test_sampler = data.sampler.SubsetRandomSampler(indices); testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers,sampler=test_sampler) else: train_dir = '/home/amir/data/imagenet12/train' test_dir = '/home/amir/data/imagenet12/val' normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform_train = transforms.Compose([transforms.RandomCrop(64),transforms.RandomHorizontalFlip(), transforms.ToTensor(),normalize]) transform_test=transforms.Compose([transforms.CenterCrop((64,64)), transforms.ToTensor(), normalize]) trainloader = DataLoader(dataset=datasets.ImageFolder(root = train_dir, transform=transform_train), batch_size=args.train_batch, shuffle=True,num_workers=1) testloader = DataLoader(dataset=datasets.ImageFolder(root = test_dir, transform=transform_test), batch_size=64, shuffle=True) # Model print("==> creating model '{}'".format(args.arch)) if 'cifar' in args.dataset: if args.arch.startswith('resnext'): model = models.__dict__[args.arch]( cardinality=args.cardinality, num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, ) elif args.arch.startswith('densenet'): if 'partial' in args.arch: model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop,part=args.part, zero_fixed_part=args.zero_fixed_part,do_init=True, split_dim = args.dim_slice ) else: model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, growthRate=args.growthRate, compressionRate=args.compressionRate, dropRate=args.drop,lateral_inhibition=args.lateral_inhibition ) elif args.arch.startswith('wrn'): if 'partial' in args.arch: print('==> initializing partial learning with p=',args.part) print('classes',num_classes,'depth',args.depth,'widen',args.widen_factor,'drop',args.drop,'part:',args.part) model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop, part=args.part, zero_fixed_part=args.zero_fixed_part ) else: model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, widen_factor=args.widen_factor, dropRate=args.drop,lateral_inhibition=args.lateral_inhibition ) elif args.arch.endswith('resnet'): model = models.__dict__[args.arch]( num_classes=num_classes, depth=args.depth, ) else: if 'partial' in args.arch: print ('PARTIAL!!!!',args.arch) #print '!!!!!!!!!!!!!!!!1' model = models.__dict__[args.arch](num_classes=num_classes,part=args.part,zero_fixed_part=args.zero_fixed_part) else: print('BOOYAH---------------!!') model = models.__dict__[args.arch](num_classes=num_classes) else: # must be imagenet if 'partial' not in args.arch: model = imagenet_models.__dict__[args.arch](depth=28, widen_factor=4, num_classes=1000) else: model = imagenet_models.__dict__[args.arch](depth=28, widen_factor=4, num_classes=1000,part=args.part,zero_fixed_part=args.zero_fixed_part) # hack hack #print('==============================', arch,'===============') #if 'squeeze' in args.arch: # model.classifier = nn.Sequential(nn.Dropout(.5), nn.Conv2d(512, 10, kernel_size=(1, 1), stride=(1, 1)), # nn.ReLU()) # print(model) from copy import deepcopy model = torch.nn.DataParallel(model).cuda() if args.load_fixed_path != '': # load the (presumably) full model dict. print('ensembling - loading old dict') fixed_model_dict = torch.load(args.load_fixed_path)['state_dict'] #if 'partial' if 'partial' in args.arch: print('transerring to new and splitting') split_model(fixed_model_dict,model) else: # just load the dictionary as is and continue from this point. model.load_state_dict(fixed_model_dict) if False: if args.load_fixed_path != '': print('ENSEMBLING') # load the fixed part of this classifier for ensembling fixed_model_dict = torch.load(args.load_fixed_path)['state_dict'] model_dict = model.state_dict() if args.part == -1: print('-------------BABU-----------------') model.load_state_dict(fixed_model_dict) elif args.only_layer != 'none': # just copy everything, the layer will be reinitialized layer. for a,b in model_dict.items(): if args.only_layer not in a: model_dict[a] = deepcopy(fixed_model_dict[a]) model.load_state_dict(model_dict) else: for a, b in model_dict.items(): # transfer all fixed values from loaded dictionary. if args.part < .5: # re-train learned part. if 'fixed' in a: model_dict[a] = deepcopy(fixed_model_dict[a]) else: # re-train what was at first the random part :-) if 'learn' in a: model_dict[a] = deepcopy(fixed_model_dict[a]) model.load_state_dict(model_dict) if args.part >= .5: # switch training between fixed / learned parts. print('HAHA, SWITCHING FIXED AND LEARNING') for a,b in model.module.named_parameters(): if 'learn' in a: b.requires_grad = False else: b.requires_grad = True # otherwise, keep it as it is. assert not (args.retrain_layer != 'none' and args.only_layer != 'none'),'retrain-layer and only-layer options are mutually exclusive' if args.retrain_layer != 'none': initial_dict = deepcopy(model.state_dict()) cudnn.benchmark = True #model = models.squeezenet1_1() # criterion = nn.CrossEntropyLoss() opt_ = args.optimizer.lower() if args.only_layer != 'none': model = only_layer(model,args.only_layer) # apply the learn-bn. for m1,m2 in model.named_modules(): if 'bn' in m1: for p in m2.parameters(): p.requires_grad = args.learn_bn if args.learn_inhibition: for p in model.module.parameters(): p.requires_grad=True params = trainableParams(model) print(' Total params: %.2fM' % (sum(p.numel() for p in model.parameters() if p.requires_grad)/1000000.0)) if opt_ == 'sgd': print('optimizer.... - sgd') optimizer = optim.SGD(params , lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) elif opt_ == 'adam': optimizer = optim.Adam(params) elif opt_ == 'yf': print('USING YF OPTIMIZER') optimizer = YFOptimizer( params, lr=args.lr, mu=0.0, weight_decay=args.weight_decay, clip_thresh=2.0, curv_win_width=20) optimizer._sparsity_debias = False else: raise Exception('unsupported optimizer type',opt_) nParamsPath = os.path.join(args.checkpoint, 'n_params.txt') with open(nParamsPath, 'w') as f: s1 = 'active_params {} \n'.format(sum(p.numel() for p in model.parameters() if p.requires_grad)) f.write(s1) s2 = 'total_params {} \n'.format(sum(p.numel() for p in model.parameters())) f.write(s2) if args.print_params_and_exit: exit() # Resume title = 'cifar-10-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoinxt..') assert os.path.isfile(args.resume), 'Error: no checkpoint directory found!' #args.checkpoint = os.path.dirname(args.checkpoint) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] #start_epoch = checkpoint['epoch'] start_epoch = args.start_epoch model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) if args.retrain_layer!='none': logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=False) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) params = trainableParams(model) print('number of trainable params:',len(list(params))) model = reinit_model_layer(model,args.retrain_layer,initial_dict) params = trainableParams(model) print('number of trainable params:',len(list(params))) optimizer = optim.SGD(params, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=False) # Was True logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val #scheduler = CosineAnnealingLR( optimizer, T_max=args.epochs)# eta_min = 1e-9, last_epoch=args.epochs) if args.half: model = model.half() for epoch in range(start_epoch, args.epochs): if args.sgdr > 0: #raise Exception('currently not supporting sgdr') scheduler.step() else: adjust_learning_rate(optimizer, epoch) if type(optimizer) is YFOptimizer: print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.get_lr_factor())) # state['lr'])) else: print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, optimizer.param_groups[0]['lr']))# state['lr'])) train_loss, train_acc = train(trainloader, model, criterion, optimizer, epoch, use_cuda) test_loss, test_acc = test(testloader, model, criterion, epoch, use_cuda) #if req_perf_after_10_epochs > -1 # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) if epoch % 10 == 0: # save each 10 epochs anyway save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint,filename='checkpoint.pth.tar_'+str(epoch).zfill(4)) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc)
def __init__(self, is_training, config, input_, opt_method='sgd'): self._input = input_ batch_size = input_.batch_size num_steps = input_.num_steps size = config.hidden_size vocab_size = config.vocab_size # Slightly better results can be obtained with forget gate biases # initialized to 1 but the hyperparameters of the model would need to be # different than reported in the paper. def lstm_cell(): # With the latest TensorFlow source code (as of Mar 27, 2017), # the BasicLSTMCell will need a reuse parameter which is unfortunately not # defined in TensorFlow 1.0. To maintain backwards compatibility, we add # an argument check here: if 'reuse' in inspect.getargspec( tf.contrib.rnn.BasicLSTMCell.__init__).args: return tf.contrib.rnn.BasicLSTMCell( size, forget_bias=0.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) else: return tf.contrib.rnn.BasicLSTMCell(size, forget_bias=0.0, state_is_tuple=True) attn_cell = lstm_cell if is_training and config.keep_prob < 1: def attn_cell(): return tf.contrib.rnn.DropoutWrapper( lstm_cell(), output_keep_prob=config.keep_prob) cell = tf.contrib.rnn.MultiRNNCell( [attn_cell() for _ in range(config.num_layers)], state_is_tuple=True) self._initial_state = cell.zero_state(batch_size, data_type()) with tf.device("cpu:0"): embedding = tf.get_variable("embedding", [vocab_size, size], dtype=data_type()) inputs = tf.nn.embedding_lookup(embedding, input_.input_data) if is_training and config.keep_prob < 1: inputs = tf.nn.dropout(inputs, config.keep_prob) # Simplified version of tensorflow.models.rnn.rnn.py's rnn(). # This builds an unrolled LSTM for tutorial purposes only. # In general, use the rnn() or state_saving_rnn() from rnn.py. # # The alternative version of the code below is: # # inputs = tf.unstack(inputs, num=num_steps, axis=1) # outputs, state = tf.nn.rnn(cell, inputs, initial_state=self._initial_state) outputs = [] state = self._initial_state with tf.variable_scope("RNN"): for time_step in range(num_steps): if time_step > 0: tf.get_variable_scope().reuse_variables() (cell_output, state) = cell(inputs[:, time_step, :], state) outputs.append(cell_output) output = tf.reshape(tf.stack(axis=1, values=outputs), [-1, size]) softmax_w = tf.get_variable("softmax_w", [size, vocab_size], dtype=data_type()) softmax_b = tf.get_variable("softmax_b", [vocab_size], dtype=data_type()) logits = tf.matmul(output, softmax_w) + softmax_b loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example( [logits], [tf.reshape(input_.targets, [-1])], [tf.ones([batch_size * num_steps], dtype=data_type())]) # self._cost = cost = tf.reduce_sum(loss) / batch_size self._cost = cost = tf.reduce_sum(loss) / (batch_size * num_steps) self._final_state = state if not is_training: return self._lr = tf.Variable(0.0, trainable=False) self._mu = tf.Variable(0.0, trainable=False) self._grad_norm_thresh = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() self.tvars = tvars self.grads = tf.gradients(cost, tvars) grads_clip, self.grad_norm = tf.clip_by_global_norm( self.grads, 100000000.0) #self._grad_norm_thresh) if opt_method == 'sgd': print("using sgd") optimizer = tf.train.GradientDescentOptimizer(self._lr) self._train_op = optimizer.apply_gradients( zip(grads_clip, tvars), global_step=tf.contrib.framework.get_or_create_global_step()) elif opt_method == 'mom': print("using sgd mom") optimizer = tf.train.MomentumOptimizer(self._lr, self._mu) self._train_op = optimizer.apply_gradients( zip(grads_clip, tvars), global_step=tf.contrib.framework.get_or_create_global_step()) elif opt_method == 'adam': print("using adam") optimizer = tf.train.AdamOptimizer(self._lr) self._train_op = optimizer.apply_gradients( zip(grads_clip, tvars), global_step=tf.contrib.framework.get_or_create_global_step()) elif opt_method == 'YF': print("using YF") #optimizer = YFOptimizer(learning_rate=1.0, momentum=0.0) #print("h max log smooth", config.h_max_log_smooth) self.optimizer = optimizer = YFOptimizer() self._train_op = optimizer.apply_gradients(zip(self.grads, tvars)) elif opt_method == "adagrad": print("using adagrad") optimizer = tf.train.AdagradOptimizer(self._lr) self._train_op = optimizer.apply_gradients( zip(grads_clip, tvars), global_step=tf.contrib.framework.get_or_create_global_step()) else: raise Exception("optimizer not supported") self._new_lr = tf.placeholder(tf.float32, shape=[], name="new_learning_rate") self._lr_update = tf.assign(self._lr, self._new_lr) self._new_mu = tf.placeholder(tf.float32, shape=[], name="new_momentum") self._mu_update = tf.assign(self._mu, self._new_mu) self._new_grad_norm_thresh = tf.placeholder( tf.float32, shape=[], name="new_grad_norm_thresh") self._grad_norm_thresh_update = tf.assign(self._grad_norm_thresh, self._new_grad_norm_thresh)
def train_eval_model(graph_hyper_params): def construct_train_data(pos_train_data, neg_train_data, graph_hyper_params): # global pos_train_data, neg_train_data, start_neg pos_len, neg_len = len(pos_train_data), len(neg_train_data) # print start_neg, pos_len, neg_len if graph_hyper_params['neg_start'] * pos_len + graph_hyper_params['neg_size'] * pos_len < neg_len: this_neg_train_data = neg_train_data[graph_hyper_params['neg_start'] * pos_len: \ graph_hyper_params['neg_start'] * pos_len + graph_hyper_params[ 'neg_size'] * pos_len] else: print 'fianl ! fianl ! fianl ! fianl !' this_neg_train_data = pd.concat([neg_train_data[graph_hyper_params['neg_start'] * pos_len:], neg_train_data[: pos_len - max(0,neg_len - graph_hyper_params['neg_start'] * pos_len)]]) train_data = pd.concat([pos_train_data, this_neg_train_data]) return shuffle(train_data) print graph_hyper_params print 'read data start !' pos_train_data, neg_train_data, predict_data1, predict_data2, user_data, ad_data, feature_conf_dict, uid_map, aid_map = get_prod_dataset(graph_hyper_params['formal']) print 'read data done !' # 重新 split train dev o_dev_size = graph_hyper_params['o_dev_size'] dev_data = pd.concat([pos_train_data[:o_dev_size], neg_train_data[:o_dev_size]]) pos_train_data, neg_train_data = pos_train_data[o_dev_size:], neg_train_data[o_dev_size:] print 'dev_size:', len(dev_data) print 'pos-neg-len:', len(pos_train_data), len(neg_train_data) train_data = construct_train_data(pos_train_data, neg_train_data, graph_hyper_params) if graph_hyper_params['only_train']: if graph_hyper_params['formal']: formal_set = set(list(train_data['uid']) + list(dev_data['uid'])) else: formal_set = set(list(train_data['uid']) + list(dev_data['uid']) + [1, 2, 3, 4]) user_data = user_data[user_data['uid'].isin(formal_set)] import gc gc.collect() print 'map row start' uid_map_row, aid_map_row = dict(zip(user_data['uid'].values, np.arange(len(user_data)))), dict(zip(ad_data['aid'].values, np.arange(len(ad_data)))) print 'map row end' print feature_conf_dict graph = tf.Graph() with graph.as_default(): # 对 creativeSize 这一个连续特征的处理 if graph_hyper_params['creativeSize_pro'] == 'min_max': print 'min-max norm creativeSize', ad_data['creativeSize'].max(), ad_data['creativeSize'].min() norm_cs = (ad_data['creativeSize'] * 1.0 - ad_data['creativeSize'].min()) / ( ad_data['creativeSize'].max() - ad_data['creativeSize'].min()) ad_data = ad_data.drop(['creativeSize'], axis=1) ad_data['creativeSize'] = norm_cs creativesize_p = tf.placeholder(tf.float32, [None, 1], name="creativeSize") elif graph_hyper_params['creativeSize_pro'] == 'li_san': print '离散化 creativeSize' sh = ShrinkSep() ad_data['creativeSize'] = ad_data['creativeSize'].apply(sh) feature_conf_dict['creativeSize'] = len(sh.d) + 1 creativesize_p = tf.placeholder(tf.int32, [None, 1], name="creativeSize") else: print 'no process creativeSize' # ****************************************************************** place holder start uid_p = tf.placeholder(tf.int32, [None, 1], name="uid") lbs_p = tf.placeholder(tf.int32, [None, 1], name="LBS") age_p = tf.placeholder(tf.int32, [None, 1], name="age") carrier_p = tf.placeholder(tf.int32, [None, 1], name="carrier") consumptionability_p = tf.placeholder(tf.int32, [None, 1], name="consumptionAbility") education_p = tf.placeholder(tf.int32, [None, 1], name="education") gender_p = tf.placeholder(tf.int32, [None, 1], name="gender") house_p = tf.placeholder(tf.int32, [None, 1], name="house") os_p = tf.placeholder(tf.int32, [None, 1], name="os") ct_p = tf.placeholder(tf.int32, [None, 1], name="ct") # marriagestatus_p = tf.placeholder(tf.int32, [None, 1], name="marriageStatus") appidaction_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['appIdAction'][1]], name="appidaction_index") appidaction_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['appIdAction'][1]], name="appidaction_val") appIdInstall_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['appIdInstall'][1]], name="appIdInstall_index") appIdInstall_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['appIdInstall'][1]], name="appIdInstall_val") marriagestatus_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['marriageStatus'][0]], name="marriageStatus_index") marriagestatus_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['marriageStatus'][0]], name="marriageStatus_val") interest1_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['interest1'][0]], name="interest1_index") interest1_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['interest1'][0]], name="interest1_val") interest2_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['interest2'][0]], name="interest2_index") interest2_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['interest2'][0]], name="interest2_val") interest3_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['interest3'][0]], name="interest3_index") interest3_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['interest3'][0]], name="interest3_val") interest4_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['interest4'][0]], name="interest4_index") interest4_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['interest4'][0]], name="interest4_val") interest5_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['interest5'][0]], name="interest5_index") interest5_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['interest5'][0]], name="interest5_val") # kmeans type # clu_200_p = tf.placeholder(tf.int32, [None, 1], name="clu_200_p") # clu_400_p = tf.placeholder(tf.int32, [None, 1], name="clu_400_p") kw1_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['kw1'][1]], name="kw1_index") kw1_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['kw1'][1]], name="kw1_val") kw2_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['kw2'][1]], name="kw2_index") kw2_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['kw2'][1]], name="kw2_val") kw3_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['kw3'][1]], name="kw3_index") kw3_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['kw3'][1]], name="kw3_val") topic1_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['topic1'][1]], name="topic1_index") topic1_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['topic1'][1]], name="topic1_val") topic2_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['topic2'][1]], name="topic2_index") topic2_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['topic2'][1]], name="topic2_val") topic3_index_p = tf.placeholder(tf.int32, [None, feature_conf_dict['topic3'][1]], name="topic3_index") topic3_val_p = tf.placeholder(tf.float32, [None, 1, feature_conf_dict['topic3'][1]], name="topic3_val") aid_p = tf.placeholder(tf.int32, [None, 1], name="aid") advertiserid_p = tf.placeholder(tf.int32, [None, 1], name="advertiserId") campaignid_p = tf.placeholder(tf.int32, [None, 1], name="campaignId") creativeid_p = tf.placeholder(tf.int32, [None, 1], name="creativeId") adcategoryid_p = tf.placeholder(tf.int32, [None, 1], name="adCategoryId") productid_p = tf.placeholder(tf.int32, [None, 1], name="productId") producttype_p = tf.placeholder(tf.int32, [None, 1], name="productType") true_label = tf.placeholder(tf.float32, [None, 1], name="true_label") # ****************************************************************** place holder end pred_val, model_loss, network_params = inference(uid_p, lbs_p, age_p, carrier_p, consumptionability_p, education_p, gender_p, house_p, os_p, ct_p, marriagestatus_index_p, marriagestatus_val_p, appidaction_index_p, appidaction_val_p, appIdInstall_index_p, appIdInstall_val_p, interest1_index_p, interest1_val_p, interest2_index_p, interest2_val_p, interest3_index_p, interest3_val_p, interest4_index_p, interest4_val_p, interest5_index_p, interest5_val_p, kw1_index_p, kw1_val_p, kw2_index_p, kw2_val_p, kw3_index_p, kw3_val_p, topic1_index_p, topic1_val_p, topic2_index_p, topic2_val_p, topic3_index_p, topic3_val_p, aid_p, advertiserid_p, campaignid_p, creativeid_p, adcategoryid_p, productid_p, producttype_p, creativesize_p, true_label, feature_conf_dict, graph_hyper_params) global_step = tf.Variable(0, name="global_step", trainable=False) train_step = None learning_rate = tf.Variable(float(graph_hyper_params['learn_rate']), trainable=False, dtype=tf.float32) learning_rate_decay_op = learning_rate.assign(learning_rate * 0.5) if graph_hyper_params['opt'] == 'adam': train_step = tf.train.AdamOptimizer(learning_rate).minimize(model_loss, global_step=global_step) elif graph_hyper_params['opt'] == 'adgrad': train_step = tf.train.AdagradOptimizer(learning_rate).minimize(model_loss, global_step=global_step) elif graph_hyper_params['opt'] == 'adadelta': train_step = tf.train.AdadeltaOptimizer(learning_rate).minimize(model_loss, global_step=global_step) elif graph_hyper_params['opt'] == 'ftrl': train_step = tf.train.FtrlOptimizer(learning_rate).minimize(model_loss, global_step=global_step) elif graph_hyper_params['opt'] == 'sgd': train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(model_loss, global_step=global_step) elif graph_hyper_params['opt'] == "yellowfin": train_step = YFOptimizer(learning_rate=learning_rate, momentum=0.0).minimize(model_loss, global_step=global_step) else: print 'No optimizer !' time_now = 'model_' + str(graph_hyper_params['model']) + datetime.now().strftime("_%Y_%m_%d_%H_%M_%S") checkpoint_dir = os.path.abspath("./checkpoints/dmf_tencent/" + time_now) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) def get_fed_dict(b_data, split_vector_data, feature_conf_dict, predict=False): if graph_hyper_params['formal']: aid_list = b_data['aid'].values uid_list = b_data['uid'].values else: if len(b_data) == 4: aid_list, uid_list = [1, 2, 3, 4], [1, 2, 3, 4] elif len(b_data) == 3: aid_list, uid_list = [1, 2, 3], [1, 2, 3] else: aid_list, uid_list = [1], [1] # print 11 # d1 = datetime.now() b_u_d, b_a_d = [], [] for b_uid in uid_list: b_u_d.append(user_data.iloc[uid_map_row[b_uid]]) for b_aid in aid_list: b_a_d.append(ad_data.iloc[aid_map_row[b_aid]]) b_u_d = pd.concat(b_u_d, axis=1).transpose() b_a_d = pd.concat(b_a_d, axis=1).transpose() # d3 = datetime.now() # print 12 # pd.concat([data.iloc[1].to_frame(), data.iloc[2].to_frame()], axis=1).transpose() fed_dict = {} fed_dict[uid_p] = np.expand_dims(b_u_d['uid'], axis=1) fed_dict[lbs_p] = np.expand_dims(b_u_d['LBS'], axis=1) fed_dict[age_p] = np.expand_dims(b_u_d['age'], axis=1) fed_dict[carrier_p] = np.expand_dims(b_u_d['carrier'], axis=1) fed_dict[consumptionability_p] = np.expand_dims(b_u_d['consumptionAbility'], axis=1) fed_dict[education_p] = np.expand_dims(b_u_d['education'], axis=1) fed_dict[gender_p] = np.expand_dims(b_u_d['gender'], axis=1) fed_dict[house_p] = np.expand_dims(b_u_d['house'], axis=1) fed_dict[os_p] = np.expand_dims(b_u_d['os'], axis=1) fed_dict[ct_p] = np.expand_dims(b_u_d['ct'], axis=1) # fed_dict[marriagestatus_p] = np.expand_dims(b_u_d['marriageStatus'], axis=1) # print 121 appidaction_li = split_vector_data(b_u_d['appIdAction']) # print 1212 fed_dict[appidaction_index_p], fed_dict[appidaction_val_p] = appidaction_li[0], appidaction_li[1] appIdInstall_li = split_vector_data(b_u_d['appIdInstall']) fed_dict[appIdInstall_index_p], fed_dict[appIdInstall_val_p] = appIdInstall_li[0], appIdInstall_li[1] # print 122 marriagestatus_li = split_vector_data(b_u_d['marriageStatus'], interest='marriageStatus', feature_config=feature_conf_dict) fed_dict[marriagestatus_index_p], fed_dict[marriagestatus_val_p] = marriagestatus_li[0], marriagestatus_li[1] interest1_li = split_vector_data(b_u_d['interest1'], interest='interest1', feature_config=feature_conf_dict) fed_dict[interest1_index_p], fed_dict[interest1_val_p] = interest1_li[0], interest1_li[1] interest2_li = split_vector_data(b_u_d['interest2'], interest='interest2', feature_config=feature_conf_dict) fed_dict[interest2_index_p], fed_dict[interest2_val_p] = interest2_li[0], interest2_li[1] interest3_li = split_vector_data(b_u_d['interest3'], interest='interest3', feature_config=feature_conf_dict) fed_dict[interest3_index_p], fed_dict[interest3_val_p] = interest3_li[0], interest3_li[1] interest4_li = split_vector_data(b_u_d['interest4'], interest='interest4', feature_config=feature_conf_dict) fed_dict[interest4_index_p], fed_dict[interest4_val_p] = interest4_li[0], interest4_li[1] interest5_li = split_vector_data(b_u_d['interest5'], interest='interest5', feature_config=feature_conf_dict) fed_dict[interest5_index_p], fed_dict[interest5_val_p] = interest5_li[0], interest5_li[1] # print 123 kw1_li = split_vector_data(b_u_d['kw1']) fed_dict[kw1_index_p], fed_dict[kw1_val_p] = kw1_li[0], kw1_li[1] kw2_li = split_vector_data(b_u_d['kw2']) fed_dict[kw2_index_p], fed_dict[kw2_val_p] = kw2_li[0], kw2_li[1] kw3_li = split_vector_data(b_u_d['kw3']) fed_dict[kw3_index_p], fed_dict[kw3_val_p] = kw3_li[0], kw3_li[1] # print 124 topic1_li = split_vector_data(b_u_d['topic1']) fed_dict[topic1_index_p], fed_dict[topic1_val_p] = topic1_li[0], topic1_li[1] topic2_li = split_vector_data(b_u_d['topic2']) fed_dict[topic2_index_p], fed_dict[topic2_val_p] = topic2_li[0], topic2_li[1] topic3_li = split_vector_data(b_u_d['topic3']) fed_dict[topic3_index_p], fed_dict[topic3_val_p] = topic3_li[0], topic3_li[1] # print 125 # # ad fed_dict[aid_p] = np.expand_dims(b_a_d['aid'], axis=1) fed_dict[advertiserid_p] = np.expand_dims(b_a_d['advertiserId'], axis=1) fed_dict[campaignid_p] = np.expand_dims(b_a_d['campaignId'], axis=1) fed_dict[creativeid_p] = np.expand_dims(b_a_d['creativeId'], axis=1) fed_dict[adcategoryid_p] = np.expand_dims(b_a_d['adCategoryId'], axis=1) fed_dict[productid_p] = np.expand_dims(b_a_d['productId'], axis=1) fed_dict[producttype_p] = np.expand_dims(b_a_d['productType'], axis=1) # print 13 # fed_dict[creativesize_p] = np.expand_dims(b_a_d['creativeSize'], axis=1) if graph_hyper_params['creativeSize_pro'] == 'min_max': fed_dict[creativesize_p] = np.expand_dims(b_a_d['creativeSize'], axis=1).astype(np.float32) elif graph_hyper_params['creativeSize_pro'] == 'li_san': fed_dict[creativesize_p] = np.expand_dims(b_a_d['creativeSize'], axis=1) else: print 'wrong feed' # label # print 14 if not predict: fed_dict[true_label] = np.expand_dims(b_data['label'].values, axis=1).astype(np.float32) # print 15 # d4 = datetime.now() # print d2-d1, d3-d2, d4-d3 # print fed_dict[true_label] # print len(fed_dict[true_label]), len(fed_dict[aid_p]), len(fed_dict[uid_p]), return fed_dict def eval_on_dev(split_vector_data): e_b_s = len(dev_data) / graph_hyper_params['batch_size'] auc_true, auc_pre = [], [] # auc = [] for index in tqdm(range(e_b_s)): start = index * graph_hyper_params['batch_size'] end = (index + 1) * graph_hyper_params['batch_size'] if (index + 1) * graph_hyper_params['batch_size'] < len(dev_data) else len(dev_data) b_dev_data = dev_data[start:end] fed_dict = get_fed_dict(b_dev_data, split_vector_data, feature_conf_dict) pred_value, pre_pred_value, final_vec, uu, vv = sess.run([pred_val, network_params[0], network_params[1], network_params[2], network_params[3]], feed_dict=fed_dict) pre_real_val = np.array(pred_value).reshape((-1)) auc_true = auc_true + list(b_dev_data['label'].values) auc_pre = auc_pre + pre_real_val.tolist() if True in np.isnan(pre_real_val): print 'contain nan: ', np.array(pre_pred_value).reshape((-1)) print np.array(final_vec).reshape((-1)) print np.array(uu).reshape((-1)) print np.array(vv).reshape((-1)) # auc.append() # auc_pre = np.array(auc_pre) # auc_pre = np.exp(auc_pre) / np.exp(auc_pre).sum() # print auc_true # print auc_pre fpr, tpr, thresholds = metrics.roc_curve(auc_true, auc_pre, pos_label=1) auc_v, gni = metrics.auc(fpr, tpr), gini_norm(auc_true, auc_pre) auc_pre_2 = np.array(auc_pre) auc_pre_2.sort() print('dev_pre_top2=%.4f %.4f min2=%.4f %.4f' % (auc_pre_2.tolist()[-1], auc_pre_2.tolist()[-2], auc_pre_2.tolist()[0], auc_pre_2.tolist()[1])) return auc_v, gni best_auc = 0.0 split_vector_data = SplitClass() sess = tf.Session() sess.run(tf.global_variables_initializer()) for epoch in range(graph_hyper_params['epoch']): # 只训练 1 轮 e_b_s = len(train_data) / graph_hyper_params['batch_size'] one_epoch_loss, one_epoch_batchnum = 0.0, 0.0 for index in tqdm(range(e_b_s)): # print 0 start = index * graph_hyper_params['batch_size'] end = (index + 1) * graph_hyper_params['batch_size'] if (index + 1) * graph_hyper_params['batch_size'] < len(train_data) else len(train_data) b_data = train_data[start:end] # print 1 # d1 = datetime.now() fed_dict = get_fed_dict(b_data, split_vector_data, feature_conf_dict) # d2 = datetime.now() # print 2 _, loss_val, pre_tr_val = sess.run([train_step, model_loss, network_params[0]], feed_dict=fed_dict) # print 3 # d3 = datetime.now() # print d2-d1, d3-d2 one_epoch_loss += loss_val one_epoch_batchnum += 1. if graph_hyper_params['debug']: print datetime.now(), index, loss_val pre_tr_val = np.array(pre_tr_val).reshape((-1)) if graph_hyper_params['debug'] or True in np.isnan(pre_tr_val): print pre_tr_val if index != 0 and index % ((e_b_s - 1) / graph_hyper_params['show_peroid']) == 0: split_vector_data.clean() auc, gn = eval_on_dev(split_vector_data) best_auc = max(auc, best_auc) format_str = '%s epoch=%.2f avg_loss=%.4f auc=%.4f best_auc=%.4f gn=%.4f' print (format_str % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (epoch + 1.0 * (index+1) / e_b_s), one_epoch_loss / one_epoch_batchnum, auc, best_auc, gn)) one_epoch_loss = one_epoch_batchnum = 0.0 # pass # predict_data = predict_data1 # if graph_hyper_params['formal']: # graph_hyper_params['batch_size'] = 1024 # e_b_s = len(predict_data) / graph_hyper_params['batch_size'] if len(predict_data) % graph_hyper_params[ # 'batch_size'] == 0 else len(predict_data) / graph_hyper_params['batch_size'] + 1 # # del split_vector_data # # gc.collect() # # split_vector_data = SplitClass() # split_vector_data.clean() # pred = [] # for index in tqdm(range(e_b_s)): # start = index * graph_hyper_params['batch_size'] # end = (index + 1) * graph_hyper_params['batch_size'] if (index + 1) * graph_hyper_params['batch_size'] < len(predict_data) else len(predict_data) + 1 # b_predict_data = predict_data[start:end] # # print len(b_predict_data), start, end # # fed_dict = get_fed_dict(b_dev_data, split_vector_data, feature_conf_dict) # fed_dict = get_fed_dict(b_predict_data, split_vector_data, feature_conf_dict, predict=True) # fed_dict[train_p] = False # fed_dict[dropout_p] = np.array([1.0]) # pred_value = sess.run([pred_val], feed_dict=fed_dict) # # print pred_value # pre_real_val = np.array(pred_value).reshape((-1)) # pred = pred + pre_real_val.tolist() # # predict_data['pred_label'] = pred # csv_data = predict_data[['ori_aid', 'ori_uid', 'pred_label']] # csv_data.columns = ['aid', 'uid', 'score'] # csv_path = os.path.join(checkpoint_dir, 'n' + str(graph_hyper_params['neg_start']) + '_submission.csv') # csv_data.to_csv(csv_path, index=False) # print 'submission_path:', csv_path pass
def _init_graph(self): self.graph = tf.Graph() with self.graph.as_default(): tf.set_random_seed(self.random_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") # None * Cate self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") # None * Num self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1 self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") self.weights = self._initialize_weights() # model self.embeddings = tf.nn.embedding_lookup( self.weights["feature_embeddings"], self.feat_index) # None * Cate * K self.embeddings = tf.reshape( self.embeddings, shape=[-1, self.cate_field_size * self.embedding_size]) self.x0 = tf.concat([self.feat_value, self.embeddings], axis=1) # ---------- Deep component ---------- self.y_deep = tf.nn.dropout(self.x0, self.dropout_keep_deep[0]) for i in range(0, len(self.deep_layers)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.weights["layer_%d" % i]), self.weights["bias_%d" % i]) # None * layer[i] * 1 if self.batch_norm: self.y_deep = self.batch_norm_layer( self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" % i) # None * layer[i] * 1 self.y_deep = self.deep_layers_activation(self.y_deep) self.y_deep = tf.nn.dropout( self.y_deep, self.dropout_keep_deep[1 + i]) # dropout at each Deep layer # ---------- Cross component ---------- self._x0 = tf.reshape(self.x0, (-1, self.total_size, 1)) x_l = self._x0 for l in range(self.cross_layer_num): x_l = tf.tensordot(tf.matmul(self._x0, x_l, transpose_b=True), self.weights["cross_layer_%d" % l], 1) + self.weights["cross_bias_%d" % l] + x_l self.cross_network_out = tf.reshape(x_l, (-1, self.total_size)) # ---------- output ---------- if self.use_cross and self.use_deep: concat_input = tf.concat([self.cross_network_out, self.y_deep], axis=1) elif self.use_cross: concat_input = self.cross_network_out elif self.use_deep: concat_input = self.y_deep self.out = tf.add( tf.matmul(concat_input, self.weights["concat_projection"]), self.weights["concat_bias"]) # loss if self.loss_type == "logloss": self.out = tf.nn.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == "mse": self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # l2 regularization on weights if self.l2_reg > 0: self.loss += tf.contrib.layers.l2_regularizer(self.l2_reg)( self.weights["concat_projection"]) if self.use_deep: for i in range(len(self.deep_layers)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["layer_%d" % i]) # optimizer if self.optimizer_type == "adam": self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss) elif self.optimizer_type == "adagrad": self.optimizer = tf.train.AdagradOptimizer( learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize(self.loss) elif self.optimizer_type == "gd": self.optimizer = tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate).minimize(self.loss) elif self.optimizer_type == "momentum": self.optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.95).minimize(self.loss) elif self.optimizer_type == "yellowfin": self.optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize(self.loss) # init self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = self._init_session() self.sess.run(init) # number of params total_parameters = 0 for variable in self.weights.values(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters if self.verbose > 0: print("#params: %d" % total_parameters)
def _init_graph(self): self.graph = tf.Graph() with self.graph.as_default(): tf.set_random_seed(self.rand_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm") self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") self.weight = self._init_weight() # 创建DeepFM模型图 self.embeddings = tf.nn.embedding_lookup( self.weight['feat_embeddings'], self.feat_index) feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1]) self.embeddings = tf.multiply(self.embeddings, feat_value) # None*F*K # 一阶项 self.y_first_order = tf.nn.embedding_lookup( self.weight["feat_bias"], self.feat_index) self.y_first_order = tf.reduce_sum( tf.multiply(self.y_first_order, feat_value), 2) self.y_first_order = tf.nn.dropout(self.y_first_order, self.dropout_keep_fm[0]) # 二阶项 # 先求和后平方 sum_feat_emb = tf.reduce_sum(self.embeddings, 1) self.sum_square_feat_emb = tf.square(sum_feat_emb) # 先平方后求和 sqrt_feat_emb = tf.square(self.embeddings) self.sqrt_sum_feat_emb = tf.reduce_sum(sqrt_feat_emb, 1) # 二阶项 self.y_second_order = 0.5 * tf.subtract(self.sum_square_feat_emb, self.sqrt_sum_feat_emb) self.y_second_order = tf.nn.dropout(self.y_second_order, self.dropout_keep_fm[1]) # deep部分 self.y_deep = tf.reshape( self.embeddings, [-1, self.field_size * self.embedding_size]) self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0]) for i in range(0, len(self.deep_layer)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.weight["layer_%d" % i]), self.weight["bias_%d" % i]) if self.batch_norm: self.y_deep = self.batch_norm_layer(self.y_deep, self.train_phase, scope_bn="bn_%d" % i) self.y_deep = self.deep_layer_activation(self.y_deep) # 三部分向量拼接 if self.use_deep and self.use_fm: concat_input = tf.concat( [self.y_first_order, self.y_second_order, self.y_deep], axis=1) elif self.use_fm: concat_input = tf.concat( [self.y_first_order, self.y_second_order], axis=1) elif self.use_deep: concat_input = self.y_deep self.out = tf.add( tf.matmul(concat_input, self.weight['concat_projection']), self.weight['concat_bias']) # 损失函数 if self.loss_type == 'logloss': self.out = tf.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == 'mse': self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # 是否加正则 if self.l2_reg > 0: self.loss += tf.contrib.layers.l2_regularizer(self.l2_reg)( self.weight["concat_projection"]) if self.use_deep: for i in range(len(self.deep_layer)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weight["layer_%d" % i]) # 优化器 if self.opt_type == 'adam': self.opt = tf.train.AdamOptimizer(self.learning_rate).minimize( self.loss) elif self.opt_type == 'adagrade': self.opt = tf.train.AdagradOptimizer( self.learning_rate).minimize(self.loss) elif self.opt_type == 'gd': self.opt = tf.train.GradientDescentOptimizer( self.learning_rate).minimize(self.loss) elif self.opt_type == 'momentum': self.opt = tf.train.MomentumOptimizer( self.learning_rate).minimize(self.loss) elif self.opt_type == 'yellowfin': self.opt = YFOptimizer( learning_rate=self.learning_rate).minimize(self.loss) # 建立init self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = self._init_session() self.sess.run(init) # 打印参数数量 total_parametres = 0 for variable in self.weight.values(): shape = variable.get_shape() variable_parametres = 1 for dim in shape: variable_parametres *= dim total_parametres += variable_parametres if self.verbose > 0: print("# params: %d" % total_parametres)
if use_cuda: net.cuda() net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count())) cudnn.benchmark = True criterion = nn.CrossEntropyLoss() if args.opt_method == "SGD": print("using SGD") optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4) elif args.opt_method == "Adam": print("using Adam") optimizer = optim.Adam(net.parameters(), lr=args.lr, weight_decay=5e-4) elif args.opt_method == "YF": print("using YF") optimizer = YFOptimizer(net.parameters(), lr=args.lr, mu=args.mu, weight_decay=5e-4) else: raise Exception("Optimizer not supported") # Training def train(epoch): print('\nEpoch: %d' % epoch) net.train() train_loss = 0 correct = 0 total = 0 loss_list = [] lr_list = [] mu_list = [] if epoch == 151: if args.opt_method == "YF":
def test_lr_mu(): opt = YFOptimizer(zero_debias=False) w = tf.Variable(np.ones([n_dim, ] ), dtype=tf.float32, name="w", trainable=True) b = tf.Variable(np.ones([1, ], dtype=np.float32), dtype=tf.float32, name="b", trainable=True) x = tf.constant(np.ones([n_dim, ], dtype=np.float32), dtype=tf.float32) loss = tf.multiply(w, x) + b tvars = tf.trainable_variables() w_grad_val = tf.Variable(np.zeros( [n_dim, ] ), dtype=tf.float32, trainable=False) b_grad_val = tf.Variable(np.zeros([1, ] ), dtype=tf.float32, trainable=False) apply_op = opt.apply_gradients(zip([w_grad_val, b_grad_val], tvars) ) init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) target_h_max = 0.0 target_h_min = 0.0 g_norm_squared_avg = 0.0 g_norm_avg = 0.0 g_avg = 0.0 target_dist = 0.0 target_lr = 0.1 target_mu = 0.0 for i in range(n_iter): sess.run(tf.assign(w_grad_val, (i + 1) * np.ones( [n_dim, ], dtype=np.float32) ) ) sess.run(tf.assign(b_grad_val, (i + 1) * np.ones( [1, ], dtype=np.float32) ) ) res = sess.run( [opt._curv_win, opt._h_max, opt._h_min, opt._grad_var, opt._dist_to_opt_avg, opt._lr_var, opt._mu_var, apply_op] ) res[5] = opt._lr_var.eval() res[6] = opt._mu_var.eval() g_norm_squared_avg = 0.999 * g_norm_squared_avg \ + 0.001 * np.sum(( (i + 1)*np.ones( [n_dim + 1, ] ) )**2) g_norm_avg = 0.999 * g_norm_avg \ + 0.001 * np.linalg.norm( (i + 1)*np.ones( [n_dim + 1, ] ) ) g_avg = 0.999 * g_avg + 0.001 * (i + 1) target_h_max = 0.999 * target_h_max + 0.001 * (i + 1)**2*(n_dim + 1) target_h_min = 0.999 * target_h_min + 0.001 * max(1, i + 2 - 20)**2*(n_dim + 1) target_var = g_norm_squared_avg - g_avg**2 * (n_dim + 1) target_dist = 0.999 * target_dist + 0.001 * g_norm_avg / g_norm_squared_avg if i > 0: lr, mu = tune_everything(target_dist**2, target_var, 1, target_h_min, target_h_max) target_lr = 0.999 * target_lr + 0.001 * lr target_mu = 0.999 * target_mu + 0.001 * mu # print "iter ", i, " h max ", res[1], target_h_max, " h min ", res[2], target_h_min, \ # " var ", res[3], target_var, " dist ", res[4], target_dist # print "iter ", i, " lr ", res[5], target_lr, " mu ", res[6], target_mu assert np.abs(target_h_max - res[1] ) < np.abs(target_h_max) * 1e-3 assert np.abs(target_h_min - res[2] ) < np.abs(target_h_min) * 1e-3 assert np.abs(target_var - res[3] ) < np.abs(res[3] ) * 1e-3 assert np.abs(target_dist - res[4] ) < np.abs(res[4] ) * 1e-3 assert target_lr == 0.0 or np.abs(target_lr - res[5] ) < np.abs(res[5] ) * 1e-3 assert target_mu == 0.0 or np.abs(target_mu - res[6] ) < np.abs(res[6] ) * 5e-3 print "lr and mu computing test passed!"
def __init__(self, is_training, config): self.batch_size = batch_size = config.batch_size self.num_steps = num_steps = config.num_steps size = config.hidden_size vocab_size = config.vocab_size self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps]) self._targets = tf.placeholder(tf.int32, [batch_size, num_steps]) # lstm_cell = tf.contrib.rnn.BasicLSTMCell(size, forget_bias=1.0, # state_is_tuple=True) # if is_training and config.keep_prob < 1: # lstm_cell = tf.contrib.rnn.DropoutWrapper( # lstm_cell, output_keep_prob=config.keep_prob) # cell = tf.contrib.rnn.MultiRNNCell([lstm_cell] * config.num_layers, # state_is_tuple=True) # Slightly better results can be obtained with forget gate biases # initialized to 1 but the hyperparameters of the model would need to be # different than reported in the paper. def lstm_cell(): # With the latest TensorFlow source code (as of Mar 27, 2017), # the BasicLSTMCell will need a reuse parameter which is unfortunately not # defined in TensorFlow 1.0. To maintain backwards compatibility, we add # an argument check here: if 'reuse' in inspect.getargspec( tf.contrib.rnn.BasicLSTMCell.__init__).args: return tf.contrib.rnn.BasicLSTMCell( size, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) else: return tf.contrib.rnn.BasicLSTMCell( size, forget_bias=1.0, state_is_tuple=True) attn_cell = lstm_cell if is_training and config.keep_prob < 1: def attn_cell(): return tf.contrib.rnn.DropoutWrapper( lstm_cell(), output_keep_prob=config.keep_prob) cell = tf.contrib.rnn.MultiRNNCell( [attn_cell() for _ in range(config.num_layers)], state_is_tuple=True) self._initial_state = cell.zero_state(batch_size, tf.float32) with tf.device("/cpu:0"): embedding = tf.get_variable("embedding", [vocab_size, size]) inputs = tf.nn.embedding_lookup(embedding, self._input_data) if is_training and config.keep_prob < 1: inputs = tf.nn.dropout(inputs, config.keep_prob) # inputs = [tf.squeeze(input_, [1]) # for input_ in tf.split(inputs, num_steps, 1)] # outputs, state = tf.contrib.rnn.static_rnn(cell, inputs, initial_state=self._initial_state) outputs = [] state = self._initial_state with tf.variable_scope("RNN"): for time_step in range(num_steps): if time_step > 0: tf.get_variable_scope().reuse_variables() (cell_output, state) = cell(inputs[:, time_step, :], state) outputs.append(cell_output) output = tf.reshape(tf.stack(axis=1, values=outputs), [-1, size]) softmax_w = tf.get_variable("softmax_w", [size, vocab_size]) softmax_b = tf.get_variable("softmax_b", [vocab_size]) logits = tf.matmul(output, softmax_w) + softmax_b loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example( [logits], [tf.reshape(self._targets, [-1])], [tf.ones([batch_size * num_steps])]) cost = tf.reduce_sum(loss) / batch_size self._norm_loss = cost / num_steps self._cost = loss self._final_state = state if not is_training: return self._lr = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self._norm_loss, tvars), config.max_grad_norm) if config.opt_method == "Adam": print("using Adam") optimizer = tf.train.AdamOptimizer(self.lr) elif config.opt_method == "YF": print("using YF") self.optimizer = optimizer = YFOptimizer(learning_rate=1.0, momentum=0.0) else: #print("uisng SGD") #optimizer = tf.train.GradientDescentOptimizer(self.lr) print("uisng mom SGD") optimizer = tf.train.MomentumOptimizer(self.lr, 0.9) self._train_op = optimizer.apply_gradients(zip(grads, tvars)) self.train_loss_summary = tf.summary.scalar('train_loss', self._norm_loss) self.writer = tf.summary.FileWriter( os.path.join(config.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S")))
dist_list,\ grad_var_list,\ lr_g_norm_list,\ lr_g_norm_squared_list,\ move_lr_g_norm_list,\ move_lr_g_norm_squared_list,\ lr_grad_norm_clamp_act_list,\ fast_view_act_list if args.opt_method == "SGD": optimizer = torch.optim.SGD(model.parameters(), lr, momentum=0.0) elif args.opt_method == "momSGD": optimizer = torch.optim.SGD(model.parameters(), lr, momentum=0.9) elif args.opt_method == "YF": optimizer = YFOptimizer(model.parameters() ) elif args.opt_method == "Adam": optimizer = torch.optim.Adam(model.parameters(), lr) best_val_loss = None train_loss_list = [] val_loss_list = [] lr_list = [] mu_list = [] loss_list = [] local_curv_list = [] max_curv_list = [] min_curv_list = [] lr_g_norm_list = []
def _init_graph(self): # 新生成的图作为整个 tensorflow 运行环境的默认图 self.graph = tf.Graph() with self.graph.as_default(): tf.set_random_seed(self.random_seed) # shape说明是一个二维矩阵,第一维是样本个数吗?像. # batch_size * 39 self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") # None * F # shape说明是一个二维矩阵: batch_size * 39 self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") # None * F self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1 self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm") self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") # 随机初始化weight self.weights = self._initialize_weights() # print(self.feat_index.shape): (1024, 39) # batch_size * 39 * 8(vi、vj的维度???感觉这样不对,这是V的shape,vi、vj的维度应该是batch_size*39*1) self.embeddings = tf.nn.embedding_lookup( self.weights["feature_embeddings"], self.feat_index ) # 之后,self.embeddings的shape:None * F * K(embbed之后的维度) # 变成了一个列向量 # 原来是 batch_size * 39====》 batch_size * 39 * 1 # reshape将shape变的跟self.embeddings的一致,为了后面两者做multiply feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1]) print(self.embeddings.shape, feat_value.shape) # batch_size * 39 * 8《== batch_size * 39 * 8,batch_size * 39 * 1 # broadcast了feature value # 这里与embedding没有什么关系了 self.embeddings = tf.multiply(self.embeddings, feat_value) # ----------wide fm: first order term ---------- # None * 39 * 1 ,如(1024, 39, 1) # 从# 259 * 1的矩阵(self.weights["feature_bias"])中找39行(self.feat_index)。 self.y_first_order = tf.nn.embedding_lookup( self.weights["feature_bias"], self.feat_index) # None * F * 1 # self.y_first_order_weights = self.y_first_order # 临时加上去的。后面删掉 self.y_first_order_tmp = self.y_first_order # self.y_first_order, feat_value都是batch_size * 39 * 1 # 算完之后就是batch_size * 39 self.y_first_order = tf.reduce_sum( tf.multiply(self.y_first_order, feat_value), 2) # None * F # 这个是一个向量,跟二阶、和dnn output出来的高阶,concatenate到一起。根据fm原理不太一样,应该output出来一个scalar才是, # 但是看concatenate之后,算了一个内积,那么也就是这个计算,作为一阶,是多余的,后面concatenate之后才是真的一阶。 # 但算了两次线性,跟一次线性计算,最终效果是一样的。 self.y_first_order = tf.nn.dropout( self.y_first_order, self.dropout_keep_fm[0]) # None * F # ----------wide fm: second order term --------------- # sum_square part # batch_size * 8 self.summed_features_emb = tf.reduce_sum(self.embeddings, 1) # None * K # batch_size * 8 self.summed_features_emb_square = tf.square( self.summed_features_emb) # None * K # square_sum part # batch_size * 8 self.squared_features_emb = tf.square(self.embeddings) # batch_size * 8 self.squared_sum_features_emb = tf.reduce_sum( self.squared_features_emb, 1) # None * K # second order # batch_size * 8。每个样本成了一个vector了 self.y_second_order = 0.5 * tf.subtract( self.summed_features_emb_square, self.squared_sum_features_emb) # None * K # batch_size * 8 self.y_second_order = tf.nn.dropout( self.y_second_order, self.dropout_keep_fm[1]) # None * K # ---------- Deep component ---------- self.y_deep = tf.reshape( self.embeddings, shape=[-1, self.field_size * self.embedding_size]) # None * (F*K) self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0]) for i in range(0, len(self.deep_layers)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.weights["layer_%d" % i]), self.weights["bias_%d" % i]) # None * layer[i] * 1 if self.batch_norm: self.y_deep = self.batch_norm_layer( self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" % i) # None * layer[i] * 1 self.y_deep = self.deep_layers_activation(self.y_deep) self.y_deep = tf.nn.dropout( self.y_deep, self.dropout_keep_deep[1 + i]) # dropout at each Deep layer # ---------- DeepFM ---------- if self.use_fm and self.use_deep: # 一阶和二阶的concat到一起 # (1024, 39)+(1024, 8)+(1024, 32)===>(1024, 79) concat_input = tf.concat( [self.y_first_order, self.y_second_order, self.y_deep], axis=1) elif self.use_fm: concat_input = tf.concat( [self.y_first_order, self.y_second_order], axis=1) elif self.use_deep: concat_input = self.y_deep self.concat_input = concat_input # 这是一个scalar neiji = tf.matmul(concat_input, self.weights["concat_projection"]) # 获得了一个scalar self.out = tf.add(neiji, self.weights["concat_bias"]) # loss if self.loss_type == "logloss": self.out = tf.nn.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == "mse": self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # l2 regularization on weights if self.l2_reg > 0: self.loss += tf.contrib.layers.l2_regularizer(self.l2_reg)( self.weights["concat_projection"]) if self.use_deep: for i in range(len(self.deep_layers)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["layer_%d" % i]) # optimizer if self.optimizer_type == "adam": self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss) elif self.optimizer_type == "adagrad": self.optimizer = tf.train.AdagradOptimizer( learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize(self.loss) elif self.optimizer_type == "gd": self.optimizer = tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate).minimize(self.loss) elif self.optimizer_type == "momentum": self.optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.95).minimize(self.loss) elif self.optimizer_type == "yellowfin": self.optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize(self.loss) # init self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = self._init_session() writer = tf.summary.FileWriter("logs/", self.sess.graph) # 第一个参数指定生成文件的目录。 self.sess.run(init) # print(self.embeddings.shape, feat_value.shape) # 统计weight个数(number of params) total_parameters = 0 for variable in self.weights.values(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters if self.verbose > 0: print("#params: %d" % total_parameters)
model.add(Activation('relu')) model.add(Conv2D(64, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes)) model.add(Activation('softmax')) # Modified to use yellowfin or adam optimizer if yellowfinopt: opt = keras.optimizers.TFOptimizer(YFOptimizer()) else: opt = keras.optimizers.Adam() # Compile the model model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 # subsamples for quick test #x_train2 = x_train[0:2000, :,:,:]
curParams = [p for p in net.parameters() if p.requires_grad] if args.optimizer.lower() == 'sgd': optimizer = optim.SGD(curParams, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) elif args.optimizer.lower() == 'adam': #args.lr=1e-3 optimizer = optim.Adam(params=curParams) elif args.optimizer.lower() == 'rmsprop': #args.lr=1e-2 optimizer = optim.RMSprop(params=curParams) elif args.optimizer.lower() in ['yellowfin', 'yf']: optimizer = YFOptimizer(curParams, lr=args.lr, mu=0.0, weight_decay=weight_decay, clip_thresh=2.0, curv_win_width=20) else: raise Exception('Unsupported optimizer type encountered:' + args.optimizer) criterion = MultiBoxLoss(num_classes, 0.5, True, 0, True, 3, 0.5, False, args.cuda) def train(): #import cProfile, pstats #from io import StringIO #pr = cProfile.Profile() #pr.enable() net.train()
def learn(dataset, rank=2, scale=1., learning_rate=1e-1, tol=1e-8, epochs=100, use_yellowfin=False, use_adagrad=False, print_freq=1, model_save_file=None, model_load_file=None, batch_size=16, num_workers=None, lazy_generation=False, log_name=None, warm_start=None, learn_scale=False, checkpoint_freq=1000, sample=1., subsample=None, exponential_rescale=None, extra_steps=1, use_svrg=False, T=10, use_hmds=False): # Log configuration formatter = logging.Formatter('%(asctime)s %(message)s') logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%FT%T', ) if log_name is not None: logging.info(f"Logging to {log_name}") log = logging.getLogger() fh = logging.FileHandler(log_name) fh.setFormatter(formatter) log.addHandler(fh) logging.info(f"Commandline {sys.argv}") if model_save_file is None: logging.warn("No Model Save selected!") G = load_graph.load_graph(dataset) GM = nx.to_scipy_sparse_matrix(G) # grab scale if warm starting: if warm_start: scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1] n = G.order() logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}") Z = None def collate(ls): x, y = zip(*ls) return torch.cat(x), torch.cat(y) if lazy_generation: if subsample is not None: z = DataLoader(GraphRowSubSampler(G, scale, subsample), batch_size, shuffle=True, collate_fn=collate) else: z = DataLoader(GraphRowSampler(G, scale), batch_size, shuffle=True, collate_fn=collate) logging.info("Built Data Sampler") else: Z = gh.build_distance(G, scale, num_workers=int(num_workers) if num_workers is not None else 16) # load the whole matrix logging.info(f"Built distance matrix with {scale} factor") if subsample is not None: z = DataLoader(GraphRowSubSampler(G, scale, subsample, Z=Z), batch_size, shuffle=True, collate_fn=collate) else: idx = torch.LongTensor([(i, j) for i in range(n) for j in range(i + 1, n)]) Z_sampled = gh.dist_sample_rebuild_pos_neg( Z, sample) if sample < 1 else Z vals = torch.DoubleTensor( [Z_sampled[i, j] for i in range(n) for j in range(i + 1, n)]) z = DataLoader(TensorDataset(idx, vals), batch_size=batch_size, shuffle=True, pin_memory=torch.cuda.is_available()) logging.info("Built data loader") if model_load_file is not None: logging.info(f"Loading {model_load_file}...") m = cudaify(torch.load(model_load_file)) logging.info( f"Loaded scale {m.scale.data[0]} {torch.sum(m.w.data)} {m.epoch}") else: logging.info(f"Creating a fresh model warm_start?={warm_start}") m_init = None if warm_start: # load from DataFrame; assume that the julia combinatorial embedding has been saved ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix() scale = ws_data[0, ws_data.shape[1] - 1] m_init = torch.DoubleTensor(ws_data[:, range(ws_data.shape[1] - 1)]) elif use_hmds: # m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,rank,scale)[1])) m_init = torch.DoubleTensor( mds_warmstart.get_model(dataset, rank, scale)[1]) logging.info( f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}" ) m = cudaify( Hyperbolic_Emb(G.order(), rank, initialize=m_init, learn_scale=learn_scale, exponential_rescale=exponential_rescale)) m.normalize() m.epoch = 0 logging.info( f"Constructed model with rank={rank} and epochs={m.epoch} isnan={np.any(np.isnan(m.w.cpu().data.numpy()))}" ) # # Build the Optimizer # # TODO: Redo this in a sensible way!! # opt = torch.optim.SGD(m.parameters(), lr=learning_rate) if use_yellowfin: from yellowfin import YFOptimizer opt = YFOptimizer(m.parameters()) if use_adagrad: opt = torch.optim.Adagrad(m.parameters()) if use_svrg: from svrg import SVRG base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD opt = SVRG(m.parameters(), lr=learning_rate, T=T, data_loader=z, opt=base_opt) logging.info(opt) # Log stats from import: when warmstarting, check that it matches Julia's stats logging.info(f"*** Initial Checkpoint. Computing Stats") major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z) logging.info("*** End Initial Checkpoint\n") for i in range(m.epoch, m.epoch + epochs): l = 0.0 m.train(True) if use_svrg: for data in z: def closure(data=data, target=None): _data = data if target is None else (data, target) c = m.loss(cu_var(_data)) c.backward() return c.data[0] l += opt.step(closure) # Projection m.normalize() else: opt.zero_grad() # This is handled by the SVRG. for the_step in range(extra_steps): # Accumulate the gradient for u in z: _loss = m.loss(cu_var(u, requires_grad=False)) _loss.backward() l += _loss.data[0] Hyperbolic_Parameter.correct_metric( m.parameters()) # NB: THIS IS THE NEW CALL # print("Scale before step: ", m.scale.data) opt.step() # print("Scale after step: ", m.scale.data) # Projection m.normalize() #l += step(m, opt, u).data[0] # Logging code if l < tol: logging.info("Found a {l} solution. Done at iteration {i}!") break if i % print_freq == 0: logging.info(f"{i} loss={l}") if i % checkpoint_freq == 0: logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving") major_stats(GM, 1 + m.scale.data[0], n, m, True, Z, z) if model_save_file is not None: fname = f"{model_save_file}.{m.epoch}" logging.info( f"Saving model into {fname} {torch.sum(m.w.data)} ") torch.save(m, fname) logging.info("*** End Major Checkpoint\n") m.epoch += 1 logging.info(f"final loss={l}") if model_save_file is not None: fname = f"{model_save_file}.final" logging.info( f"Saving model into {fname}-final {torch.sum(m.w.data)} {m.scale.data[0]}" ) torch.save(m, fname) major_stats(GM, 1 + m.scale.data[0], n, m, lazy_generation, Z, z)
# mix_softmax = MixtureSoftmax(batch_size=batch_size, word_gru_hidden = 50, feature_dim = 0, n_classes=num_classes) if use_cuda: word_attn.cuda() mix_softmax.cuda() softmax = nn.Softmax() sigmoid = nn.Sigmoid() learning_rate = 0.0001 print("lr thresh", args.lr_thresh) optimizer = YFOptimizer(mix_softmax.parameters(), beta=0.999, lr=learning_rate, mu=0.0, zero_debias=False, clip_thresh=None, auto_clip_fac=None, curv_win_width=20, force_non_inc_step=False, use_disk_checkpoint=False) # word_optmizer = YFOptimizer(word_attn.parameters(), lr=learning_rate, mu=0.0, auto_clip_fac=2.0) # mix_optimizer = YFOptimizer(mix_softmax.parameters(), lr=learning_rate, mu=0.0, auto_clip_fac=2.0) criterion = nn.MultiLabelSoftMarginLoss(size_average=True) import time import math def timeSince(since):
def test_measurement(zero_debias=True): dtype = torch.FloatTensor w = Variable(torch.ones(n_dim, 1).type(dtype), requires_grad=True) b = Variable(torch.ones(1).type(dtype), requires_grad=True) x = Variable(torch.ones(1, n_dim).type(dtype), requires_grad=False) opt = YFOptimizer([w, b], lr=1.0, mu=0.0, zero_debias=zero_debias) target_h_max = 0.0 target_h_min = 0.0 g_norm_squared_avg = 0.0 g_norm_avg = 0.0 g_avg = 0.0 target_dist = 0.0 for i in range(n_iter): opt.zero_grad() loss = (x.mm(w) + b).sum() loss.backward() w.grad.data = (i + 1) * torch.ones([ n_dim, ]).type(dtype) b.grad.data = (i + 1) * torch.ones([ 1, ]).type(dtype) opt.step() res = [opt._h_max, opt._h_min, opt._grad_var, opt._dist_to_opt] g_norm_squared_avg = 0.999 * g_norm_squared_avg \ + 0.001 * np.sum(((i + 1) * np.ones([n_dim + 1, ]))**2) g_norm_avg = 0.999 * g_norm_avg \ + 0.001 * np.linalg.norm((i + 1) * np.ones([n_dim + 1, ])) g_avg = 0.999 * g_avg + 0.001 * (i + 1) target_h_max = 0.999 * target_h_max + 0.001 * (i + 1)**2 * (n_dim + 1) target_h_min = 0.999 * target_h_min + 0.001 * \ max(1, i + 2 - 20)**2 * (n_dim + 1) if zero_debias: target_var = g_norm_squared_avg / (1 - 0.999**(i + 1)) \ - g_avg**2 * (n_dim + 1) / (1 - 0.999**(i + 1))**2 else: target_var = g_norm_squared_avg - g_avg**2 * (n_dim + 1) target_dist = 0.999 * target_dist + 0.001 * g_norm_avg / g_norm_squared_avg if i == 0: continue if zero_debias: # print "iter ", i, " h max ", res[0], target_h_max/(1-0.999**(i + 1) ), \ # " h min ", res[1], target_h_min/(1-0.999**(i + 1) ), \ # " var ", res[2], target_var, \ # " dist ", res[3], target_dist/(1-0.999**(i + 1) ) assert np.abs(target_h_max / (1 - 0.999**(i + 1)) - res[0]) < np.abs(res[0]) * 1e-3 assert np.abs(target_h_min / (1 - 0.999**(i + 1)) - res[1]) < np.abs(res[1]) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(target_var) * 1e-3 assert np.abs(target_dist / (1 - 0.999**(i + 1)) - res[3]) < np.abs(res[3]) * 1e-3 else: # print "iter ", i, " h max ", res[0], target_h_max, " h min ", res[1], target_h_min, \ # " var ", res[2], target_var, " dist ", res[3], target_dist assert np.abs(target_h_max - res[0]) < np.abs(target_h_max) * 1e-3 assert np.abs(target_h_min - res[1]) < np.abs(target_h_min) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(res[2]) * 1e-3 assert np.abs(target_dist - res[3]) < np.abs(res[3]) * 1e-3 print "sync measurement test passed!"
def test_lr_mu(zero_debias=False): dtype = torch.FloatTensor w = Variable(torch.ones(n_dim, 1).type(dtype), requires_grad=True) b = Variable(torch.ones(1).type(dtype), requires_grad=True) x = Variable(torch.ones(1, n_dim).type(dtype), requires_grad=False) opt = YFOptimizer([w, b], lr=1.0, mu=0.0, zero_debias=zero_debias) target_h_max = 0.0 target_h_min = 0.0 g_norm_squared_avg = 0.0 g_norm_avg = 0.0 g_avg = 0.0 target_dist = 0.0 target_lr = 1.0 target_mu = 0.0 for i in range(n_iter): opt.zero_grad() loss = (x.mm(w) + b).sum() loss.backward() w.grad.data = (i + 1) * torch.ones([ n_dim, ]).type(dtype) b.grad.data = (i + 1) * torch.ones([ 1, ]).type(dtype) opt.step() res = [ opt._h_max, opt._h_min, opt._grad_var, opt._dist_to_opt, opt._lr, opt._mu ] g_norm_squared_avg = 0.999 * g_norm_squared_avg \ + 0.001 * np.sum(((i + 1) * np.ones([n_dim + 1, ]))**2) g_norm_avg = 0.999 * g_norm_avg \ + 0.001 * np.linalg.norm((i + 1) * np.ones([n_dim + 1, ])) g_avg = 0.999 * g_avg + 0.001 * (i + 1) target_h_max = 0.999 * target_h_max + 0.001 * (i + 1)**2 * (n_dim + 1) target_h_min = 0.999 * target_h_min + 0.001 * \ max(1, i + 2 - 20)**2 * (n_dim + 1) if zero_debias: target_var = g_norm_squared_avg / (1 - 0.999**(i + 1)) \ - g_avg**2 * (n_dim + 1) / (1 - 0.999**(i + 1))**2 else: target_var = g_norm_squared_avg - g_avg**2 * (n_dim + 1) target_dist = 0.999 * target_dist + 0.001 * g_norm_avg / g_norm_squared_avg if i == 0: continue if zero_debias: # print "iter ", i, " h max ", res[0], target_h_max/(1-0.999**(i + 1) ), \ # " h min ", res[1], target_h_min/(1-0.999**(i + 1) ), \ # " var ", res[2], target_var, \ # " dist ", res[3], target_dist/(1-0.999**(i + 1) ) assert np.abs(target_h_max / (1 - 0.999**(i + 1)) - res[0]) < np.abs(res[0]) * 1e-3 assert np.abs(target_h_min / (1 - 0.999**(i + 1)) - res[1]) < np.abs(res[1]) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(target_var) * 1e-3 assert np.abs(target_dist / (1 - 0.999**(i + 1)) - res[3]) < np.abs(res[3]) * 1e-3 else: # print "iter ", i, " h max ", res[0], target_h_max, " h min ", res[1], target_h_min, \ # " var ", res[2], target_var, " dist ", res[3], target_dist assert np.abs(target_h_max - res[0]) < np.abs(target_h_max) * 1e-3 assert np.abs(target_h_min - res[1]) < np.abs(target_h_min) * 1e-3 assert np.abs(target_var - res[2]) < np.abs(res[2]) * 1e-3 assert np.abs(target_dist - res[3]) < np.abs(res[3]) * 1e-3 if i > 0: if zero_debias: lr, mu = tune_everything( (target_dist / (1 - 0.999**(i + 1)))**2, target_var, 1, target_h_min / (1 - 0.999**(i + 1)), target_h_max / (1 - 0.999**(i + 1))) else: lr, mu = tune_everything(target_dist**2, target_var, 1, target_h_min, target_h_max) lr = np.real(lr) mu = np.real(mu) target_lr = 0.999 * target_lr + 0.001 * lr target_mu = 0.999 * target_mu + 0.001 * mu # print "lr ", target_lr, res[4], " mu ", target_mu, res[5] assert target_lr == 0.0 or np.abs(target_lr - res[4]) < np.abs(res[4]) * 1e-3 assert target_mu == 0.0 or np.abs(target_mu - res[5]) < np.abs(res[5]) * 5e-3 print "lr and mu computing test passed!"
def learn(dataset, dim=2, hyp=1, edim=1, euc=0, sdim=1, sph=0, scale=1., riemann=False, learning_rate=1e-1, decay_length=1000, decay_step=1.0, momentum=0.0, tol=1e-8, epochs=100, burn_in=0, use_yellowfin=False, use_adagrad=False, resample_freq=1000, print_freq=1, model_save_file=None, model_load_file=None, batch_size=16, num_workers=None, lazy_generation=False, log_name=None, log=False, warm_start=None, learn_scale=False, checkpoint_freq=100, sample=1., subsample=None, logloss=False, distloss=False, squareloss=False, symloss=False, exponential_rescale=None, extra_steps=1, use_svrg=False, T=10, use_hmds=False, visualize=False): # Log configuration formatter = logging.Formatter('%(asctime)s %(message)s') logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%FT%T', ) if log_name is None and log: log_name = f"{os.path.splitext(dataset)[0]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.log" if log_name is not None: logging.info(f"Logging to {log_name}") log = logging.getLogger() fh = logging.FileHandler(log_name) fh.setFormatter(formatter) log.addHandler(fh) ############# loss_list = [] ############# logging.info(f"Commandline {sys.argv}") if model_save_file is None: logging.warning("No Model Save selected!") G = load_graph.load_graph(dataset) GM = nx.to_scipy_sparse_matrix(G, nodelist=list(range(G.order()))) # grab scale if warm starting: if warm_start: scale = pandas.read_csv(warm_start, index_col=0).as_matrix()[0, -1] n = G.order() logging.info(f"Loaded Graph {dataset} with {n} nodes scale={scale}") if exponential_rescale is not None: # torch.exp(exponential_rescale * -d) def weight_fn(d): if d <= 2.0: return 5.0 elif d > 4.0: return 0.01 else: return 1.0 else: def weight_fn(d): return 1.0 Z, z = build_dataset(G, lazy_generation, sample, subsample, scale, batch_size, weight_fn, num_workers) if model_load_file is not None: logging.info(f"Loading {model_load_file}...") m = torch.load(model_load_file).to(device) logging.info( f"Loaded scale {unwrap(m.scale())} {torch.sum(m.embedding().data)} {m.epoch}" ) else: logging.info(f"Creating a fresh model warm_start?={warm_start}") m_init = None if warm_start: # load from DataFrame; assume that the julia combinatorial embedding has been saved ws_data = pandas.read_csv(warm_start, index_col=0).as_matrix() scale = ws_data[0, ws_data.shape[1] - 1] m_init = torch.DoubleTensor(ws_data[:, range(ws_data.shape[1] - 1)]) elif use_hmds: # m_init = torch.DoubleTensor(mds_warmstart.get_normalized_hyperbolic(mds_warmstart.get_model(dataset,dim,scale)[1])) m_init = torch.DoubleTensor( mds_warmstart.get_model(dataset, dim, scale)[1]) logging.info( f"\t Warmstarting? {warm_start} {m_init.size() if warm_start else None} {G.order()}" ) # initial_scale = z.dataset.max_dist / 3.0 # print("MAX DISTANCE", z.dataset.max_dist) # print("AVG DISTANCE", torch.mean(z.dataset.val_cache)) initial_scale = 0.0 m = ProductEmbedding(G.order(), dim, hyp, edim, euc, sdim, sph, initialize=m_init, learn_scale=learn_scale, initial_scale=initial_scale, logrel_loss=logloss, dist_loss=distloss, square_loss=squareloss, sym_loss=symloss, exponential_rescale=exponential_rescale, riemann=riemann).to(device) m.normalize() m.epoch = 0 logging.info( f"Constructed model with dim={dim} and epochs={m.epoch} isnan={np.any(np.isnan(m.embedding().cpu().data.numpy()))}" ) if visualize: name = 'animations/' + f"{os.path.split(os.path.splitext(dataset)[0])[1]}.H{dim}-{hyp}.E{edim}-{euc}.S{sdim}-{sph}.lr{learning_rate}.ep{epochs}.seed{seed}" fig, ax, writer = vis.setup_plot(m=m, name=name, draw_circle=True) else: fig = None ax = None writer = None # # Build the Optimizer # # TODO: Redo this in a sensible way!! # per-parameter learning rates exp_params = [p for p in m.embed_params if p.use_exp] learn_params = [p for p in m.embed_params if not p.use_exp] hyp_params = [p for p in m.hyp_params if not p.use_exp] euc_params = [p for p in m.euc_params if not p.use_exp] sph_params = [p for p in m.sph_params if not p.use_exp] scale_params = m.scale_params # model_params = [{'params': m.embed_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}] # model_params = [{'params': learn_params}, {'params': m.scale_params, 'lr': 1e-4*learning_rate}] model_params = [{ 'params': hyp_params }, { 'params': euc_params }, { 'params': sph_params, 'lr': 0.1 * learning_rate }, { 'params': m.scale_params, 'lr': 1e-4 * learning_rate }] # opt = None if len(model_params) > 0: opt = torch.optim.SGD(model_params, lr=learning_rate / 10, momentum=momentum) # opt = torch.optim.SGD(learn_params, lr=learning_rate/10, momentum=momentum) # opt = torch.optim.SGD(model_params, lr=learning_rate/10, momentum=momentum) # exp = None # if len(exp_params) > 0: # exp = torch.optim.SGD(exp_params, lr=1.0) # dummy for zeroing if len(scale_params) > 0: scale_opt = torch.optim.SGD(scale_params, lr=1e-3 * learning_rate) scale_decay = torch.optim.lr_scheduler.StepLR(scale_opt, step_size=1, gamma=.99) else: scale_opt = None scale_decay = None lr_burn_in = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[burn_in], gamma=10) # lr_decay = torch.optim.lr_scheduler.StepLR(opt, decay_length, decay_step) #TODO reconcile multiple LR schedulers if use_yellowfin: from yellowfin import YFOptimizer opt = YFOptimizer(model_params) if use_adagrad: opt = torch.optim.Adagrad(model_params) if use_svrg: from svrg import SVRG base_opt = torch.optim.Adagrad if use_adagrad else torch.optim.SGD opt = SVRG(m.parameters(), lr=learning_rate, T=T, data_loader=z, opt=base_opt) # TODO add ability for SVRG to take parameter groups logging.info(opt) # Log stats from import: when warmstarting, check that it matches Julia's stats logging.info(f"*** Initial Checkpoint. Computing Stats") major_stats(GM, n, m, lazy_generation, Z, z, fig, ax, writer, visualize, subsample) logging.info("*** End Initial Checkpoint\n") # track best stats best_loss = 1.0e10 best_dist = 1.0e10 best_wcdist = 1.0e10 best_map = 0.0 for i in range(m.epoch + 1, m.epoch + epochs + 1): lr_burn_in.step() # lr_decay.step() # scale_decay.step() # print(scale_opt.param_groups[0]['lr']) # for param_group in opt.param_groups: # print(param_group['lr']) # print(type(opt.param_groups), opt.param_groups) l, n_edges = 0.0, 0.0 # track average loss per edge m.train(True) if use_svrg: for data in z: def closure(data=data, target=None): _data = data if target is None else (data, target) c = m.loss(_data.to(device)) c.backward() return c.data[0] l += opt.step(closure) # Projection m.normalize() else: # scale_opt.zero_grad() for the_step in range(extra_steps): # Accumulate the gradient for u in z: # Zero out the gradients # if opt is not None: opt.zero_grad() # This is handled by the SVRG. # if exp is not None: exp.zero_grad() opt.zero_grad() for p in exp_params: if p.grad is not None: p.grad.detach_() p.grad.zero_() # Compute loss _loss = m.loss(cu_var(u)) _loss.backward() l += _loss.item() * u[0].size(0) # print(weight) n_edges += u[0].size(0) # modify gradients if necessary RParameter.correct_metric(m.parameters()) # step opt.step() for p in exp_params: lr = opt.param_groups[0]['lr'] p.exp(lr) # Projection m.normalize() # scale_opt.step() l /= n_edges # m.epoch refers to num of training epochs finished m.epoch += 1 # Logging code # if l < tol: # logging.info("Found a {l} solution. Done at iteration {i}!") # break if i % print_freq == 0: logging.info(f"{i} loss={l}") ############ if log_name is not None: loss_list.append(l) ############# if (i <= burn_in and i % (checkpoint_freq / 5) == 0) or i % checkpoint_freq == 0: logging.info(f"\n*** Major Checkpoint. Computing Stats and Saving") avg_dist, wc_dist, me, mc, mapscore = major_stats( GM, n, m, True, Z, z, fig, ax, writer, visualize, subsample) best_loss = min(best_loss, l) best_dist = min(best_dist, avg_dist) best_wcdist = min(best_wcdist, wc_dist) best_map = max(best_map, mapscore) if model_save_file is not None: fname = f"{model_save_file}.{m.epoch}" logging.info( f"Saving model into {fname} {torch.sum(m.embedding().data)} " ) torch.save(m, fname) logging.info("*** End Major Checkpoint\n") if i % resample_freq == 0: if sample < 1. or subsample is not None: Z, z = build_dataset(G, lazy_generation, sample, subsample, scale, batch_size, weight_fn, num_workers) logging.info(f"final loss={l}") logging.info( f"best loss={best_loss}, distortion={best_dist}, map={best_map}, wc_dist={best_wcdist}" ) final_dist, final_wc, final_me, final_mc, final_map = major_stats( GM, n, m, lazy_generation, Z, z, fig, ax, writer, False, subsample) if log_name is not None: ### with open(log_name + "_loss.stat", "w") as f: for loss in loss_list: f.write("%f\n" % loss) ### with open(log_name + '_final.stat', "w") as f: f.write("Best-loss MAP dist wc Final-loss MAP dist wc me mc\n") f.write( f"{best_loss:10.6f} {best_map:8.4f} {best_dist:8.4f} {best_wcdist:8.4f} {l:10.6f} {final_map:8.4f} {final_dist:8.4f} {final_wc:8.4f} {final_me:8.4f} {final_mc:8.4f}" ) if visualize: writer.finish() if model_save_file is not None: fname = f"{model_save_file}.final" logging.info( f"Saving model into {fname}-final {torch.sum(m.embedding().data)} {unwrap(m.scale())}" ) torch.save(m, fname)
def _init_graph(self): self.graph = tf.Graph() with self.graph.as_default(): tf.set_random_seed(self.random_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") # None * F self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") # None * F self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1 self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm") self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") self.weights = self._initialize_weights() # model self.embeddings = tf.nn.embedding_lookup( self.weights["feature_embeddings"], self.feat_index) # None * F * K feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1]) self.embeddings = tf.multiply(self.embeddings, feat_value) # ---------- first order term ---------- self.y_first_order = tf.nn.embedding_lookup( self.weights["feature_bias"], self.feat_index) # None * F * 1 self.y_first_order = tf.reduce_sum( tf.multiply(self.y_first_order, feat_value), 2) # None * F self.y_first_order = tf.nn.dropout( self.y_first_order, self.dropout_keep_fm[0]) # None * F # ---------- second order term --------------- # sum_square part self.summed_features_emb = tf.reduce_sum(self.embeddings, 1) # None * K self.summed_features_emb_square = tf.square( self.summed_features_emb) # None * K # square_sum part self.squared_features_emb = tf.square(self.embeddings) self.squared_sum_features_emb = tf.reduce_sum( self.squared_features_emb, 1) # None * K # second order self.y_second_order = 0.5 * tf.subtract( self.summed_features_emb_square, self.squared_sum_features_emb) # None * K self.y_second_order = tf.nn.dropout( self.y_second_order, self.dropout_keep_fm[1]) # None * K # ---------- Deep component ---------- self.y_deep = tf.reshape( self.embeddings, shape=[-1, self.field_size * self.embedding_size]) # None * (F*K) self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0]) for i in range(0, len(self.deep_layers)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.weights["layer_%d" % i]), self.weights["bias_%d" % i]) # None * layer[i] * 1 if self.batch_norm: self.y_deep = self.batch_norm_layer( self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" % i) # None * layer[i] * 1 self.y_deep = self.deep_layers_activation(self.y_deep) self.y_deep = tf.nn.dropout( self.y_deep, self.dropout_keep_deep[1 + i]) # dropout at each Deep layer # ---------- DeepFM ---------- if self.use_fm and self.use_deep: concat_input = tf.concat( [self.y_first_order, self.y_second_order, self.y_deep], axis=1) elif self.use_fm: concat_input = tf.concat( [self.y_first_order, self.y_second_order], axis=1) elif self.use_deep: concat_input = self.y_deep self.out = tf.add( tf.matmul(concat_input, self.weights["concat_projection"]), self.weights["concat_bias"]) # loss if self.loss_type == "logloss": self.out = tf.nn.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == "mse": self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # l2 regularization on weights if self.l2_reg > 0: self.loss += tf.contrib.layers.l2_regularizer(self.l2_reg)( self.weights["concat_projection"]) if self.use_deep: for i in range(len(self.deep_layers)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["layer_%d" % i]) # optimizer if self.optimizer_type == "adam": self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss) elif self.optimizer_type == "adagrad": self.optimizer = tf.train.AdagradOptimizer( learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize(self.loss) elif self.optimizer_type == "gd": self.optimizer = tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate).minimize(self.loss) elif self.optimizer_type == "momentum": self.optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.95).minimize(self.loss) elif self.optimizer_type == "yellowfin": self.optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize(self.loss) # init self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = self._init_session() self.sess.run(init) # number of params total_parameters = 0 for variable in self.weights.values(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters if self.verbose > 0: print("#params: %d" % total_parameters)
# Use the nn package to define our model and loss function. model = torch.nn.Sequential( torch.nn.Linear(D_in, H), torch.nn.ReLU(), torch.nn.Linear(H, D_out), ) loss_fn = torch.nn.MSELoss(size_average=False) # Use the optim package to define an Optimizer that will update the weights of # the model for us. Here we will use Adam; the optim package contains many other # optimization algoriths. The first argument to the Adam constructor tells the # optimizer which Variables it should update. min_loss_so_far = np.inf optimizer = YFOptimizer(model.parameters(), lr=0.0001, mu=0.0) for t in range(6600): # Forward pass: compute predicted y by passing x to the model. y_pred = model(x) # Compute and print loss. loss = loss_fn(y_pred, y) # Before the backward pass, use the optimizer object to zero all of the # gradients for the variables it will update (which are the learnable weights # of the model) optimizer.zero_grad() # Backward pass: compute gradient of the loss with respect to model parameters loss.backward()
def __init__(self, args, training=True, opt_method="Adam"): self.args = args if not training: args.batch_size = 1 args.seq_length = 1 if args.model == 'rnn': cell_fn = rnn.BasicRNNCell elif args.model == 'gru': cell_fn = rnn.GRUCell elif args.model == 'lstm': cell_fn = rnn.BasicLSTMCell elif args.model == 'nas': cell_fn = rnn.NASCell else: raise Exception("model type not supported: {}".format(args.model)) cells = [] for _ in range(args.num_layers): cell = cell_fn(args.rnn_size) if training and (args.output_keep_prob < 1.0 or args.input_keep_prob < 1.0): cell = rnn.DropoutWrapper( cell, input_keep_prob=args.input_keep_prob, output_keep_prob=args.output_keep_prob) cells.append(cell) self.cell = cell = rnn.MultiRNNCell(cells, state_is_tuple=True) self.input_data = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.targets = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.initial_state = cell.zero_state(args.batch_size, tf.float32) with tf.variable_scope('rnnlm'): softmax_w = tf.get_variable("softmax_w", [args.rnn_size, args.vocab_size]) softmax_b = tf.get_variable("softmax_b", [args.vocab_size]) embedding = tf.get_variable("embedding", [args.vocab_size, args.rnn_size]) inputs = tf.nn.embedding_lookup(embedding, self.input_data) # dropout beta testing: double check which one should affect next line if training and args.output_keep_prob: inputs = tf.nn.dropout(inputs, args.output_keep_prob) inputs = tf.split(inputs, args.seq_length, 1) inputs = [tf.squeeze(input_, [1]) for input_ in inputs] def loop(prev, _): prev = tf.matmul(prev, softmax_w) + softmax_b prev_symbol = tf.stop_gradient(tf.argmax(prev, 1)) return tf.nn.embedding_lookup(embedding, prev_symbol) outputs, last_state = legacy_seq2seq.rnn_decoder( inputs, self.initial_state, cell, loop_function=loop if not training else None, scope='rnnlm') output = tf.reshape(tf.concat(outputs, 1), [-1, args.rnn_size]) self.logits = tf.matmul(output, softmax_w) + softmax_b self.probs = tf.nn.softmax(self.logits) loss = legacy_seq2seq.sequence_loss_by_example( [self.logits], [tf.reshape(self.targets, [-1])], [tf.ones([args.batch_size * args.seq_length])]) self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length with tf.name_scope('cost'): self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length # for eval visualization in tensorboard self.eval_cost = tf.identity(self.cost) self.final_state = last_state self.lr = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), args.grad_clip) with tf.name_scope('optimizer'): if opt_method == "Adam": print("using Adam") self.optimizer = optimizer = tf.train.AdamOptimizer(self.lr) elif opt_method == "YF": print("using YF") self.optimizer = optimizer = YFOptimizer( learning_rate=args.learning_rate, momentum=0.0) elif opt_method == "SGD": print("using SGD") self.optimizer = optimizer = tf.train.MomentumOptimizer( self.lr, 0.9) else: raise Exception("please use either adam or YF") self.train_op = optimizer.apply_gradients(zip(grads, tvars)) # instrument tensorboard self.train_summary = [ \ tf.summary.histogram('logits', self.logits), tf.summary.histogram('loss', loss), tf.summary.scalar('train_loss', self.cost) ]
if not os.path.isdir(args.logdir): os.mkdir(args.logdir) train_loss_list = [] val_loss_list = [] lr_list = [] mu_list = [] if args.opt_method == "SGD": print("using SGD") optimizer = torch.optim.SGD(model.parameters(), lr, momentum=0.0) elif args.opt_method == "momSGD": print("using mom SGD") optimizer = torch.optim.SGD(model.parameters(), lr, momentum=0.9) elif args.opt_method == "YF": print("using YF") optimizer = YFOptimizer(model.parameters(), lr=1.0, mu=0.0) elif args.opt_method == "Adagrad": print("using Adagrad") optimizer = torch.optim.Adagrad(model.parameters(), lr) elif args.opt_method == "Adam": print("using Adam") optimizer = torch.optim.Adam(model.parameters(), lr) for epoch in range(1, args.epochs+1): epoch_start_time = time.time() train_loss = train() train_loss_list += train_loss val_loss = evaluate(val_data) val_loss_list.append(val_loss) print('-' * 89) print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | ' 'valid ppl {:8.2f}'.format(epoch,
if use_cuda: word_attn.cuda() mix_softmax.cuda() # In[9]: softmax = nn.Softmax() sigmoid = nn.Sigmoid() # In[10]: print("lr thresh", args.lr_thresh) optimizer = YFOptimizer(mix_softmax.parameters(), beta=0.999, zero_debias=False, clip_thresh=None, verbose=True, auto_clip_fac=None, curv_win_width=20, force_non_inc_step=False) # word_optmizer = YFOptimizer(word_attn.parameters(), lr=learning_rate, mu=0.0, auto_clip_fac=2.0) # mix_optimizer = YFOptimizer(mix_softmax.parameters(), lr=learning_rate, mu=0.0, auto_clip_fac=2.0) criterion = nn.MultiLabelSoftMarginLoss(size_average=True) # In[11]: #if use_cuda: # word_attn.cuda() # mix_softmax.cuda() # In[12]:
def _init_graph(self): self.graph = tf.Graph() with self.graph.as_default(): # todo tf.set_random_seed()函数,使用之后后面设置的随机数都不需要设置seed,而可以跨会话生成相同的随机数。 tf.set_random_seed(self.random_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") # None * F self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") # None * F self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1 self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm") self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") self.weights = self._initialize_weights() # model # todo 下面这一步就是根据原始特征和embedding矩阵相乘得到的每个特征的embedding。 # embedding的形成方式,首先经过FM也好Deep也好得到一个size为(M,k)的权重矩阵。这里field是尚未做onehot前特征矩阵的一维, # onehot后每个field只有1个数值不为0,这个不为0特征对应的(M,k)的权重矩阵就是这个field的embedding,这里说的都是类别类型特征。 # 开始对field有些误解,说一下自己对类别类型特征的embedding的形成方式的理解:首先经过FM也好Deep也好得到一个size为(M,K)的权重矩阵。这里field是尚未做onehot前特征矩阵的一个特征,onehot后每个field只有一个数值不为0(等于1),这个等于1对应的(M,k)的权重矩阵的一行就是这个field的embedding。 self.embeddings = tf.nn.embedding_lookup( self.weights["feature_embeddings"], self.feat_index) # None * F * K feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1]) # todo self.embeddings=Tensor("embedding_lookup/Identity:0", shape=(?, ?, 8), dtype=float32) # print(f"self.embeddings={self.embeddings}") # todo multiply 是元素级别的相乘,也就是两个相乘的数元素各自相乘,而不是矩阵乘法,注意和tf.matmul区别。 # 这里embedding和feat_value相乘主要是考虑到数值型特征,类别型特征value都是1乘不乘都一样。 self.embeddings = tf.multiply(self.embeddings, feat_value) # todo self.embeddings=Tensor("Mul:0", shape=(?, 39, 8), dtype=float32), # feat_value=Tensor("Reshape:0", shape=(?, 39, 1), dtype=float32) # print(f"self.embeddings={self.embeddings},feat_value={feat_value}") # ---------- first order term ---------- self.y_first_order = tf.nn.embedding_lookup( self.weights["feature_bias"], self.feat_index) # None * F * 1 # todo 这里为什么会喂入稀疏矩阵 self.y_first_order = tf.reduce_sum( tf.multiply(self.y_first_order, feat_value), 2) # None * F self.y_first_order = tf.nn.dropout( self.y_first_order, self.dropout_keep_fm[0]) # None * F # ---------- second order term --------------- # todo FM的输入确实是稀疏矩阵,但是DeepFM的图中画不是FM的输入是稠密矩阵吗? # FM的输入是稀疏矩阵,deep输入是稠密矩阵(这里deep的稠密矩阵是基于FM的矩阵分解得到的对吧?) # sum_square part self.summed_features_emb = tf.reduce_sum(self.embeddings, 1) # None * K self.summed_features_emb_square = tf.square( self.summed_features_emb) # None * K # square_sum part self.squared_features_emb = tf.square(self.embeddings) self.squared_sum_features_emb = tf.reduce_sum( self.squared_features_emb, 1) # None * K # second order self.y_second_order = 0.5 * tf.subtract( self.summed_features_emb_square, self.squared_sum_features_emb) # None * K self.y_second_order = tf.nn.dropout( self.y_second_order, self.dropout_keep_fm[1]) # None * K # ---------- Deep component ---------- # todo 这里反向传播的时候会更新embedding的值吗?-- 应该会更新吧,如果只使用deep的时候肯定会更新,FM和deep一起使用怎么会不更新呢? self.y_deep = tf.reshape( self.embeddings, shape=[-1, self.field_size * self.embedding_size]) # None * (F*K) self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0]) for i in range(0, len(self.deep_layers)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.weights["layer_%d" % i]), self.weights["bias_%d" % i]) # None * layer[i] * 1 if self.batch_norm: self.y_deep = self.batch_norm_layer( self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" % i) # None * layer[i] * 1 self.y_deep = self.deep_layers_activation(self.y_deep) self.y_deep = tf.nn.dropout( self.y_deep, self.dropout_keep_deep[1 + i]) # dropout at each Deep layer # ---------- DeepFM ---------- if self.use_fm and self.use_deep: concat_input = tf.concat( [self.y_first_order, self.y_second_order, self.y_deep], axis=1) elif self.use_fm: concat_input = tf.concat( [self.y_first_order, self.y_second_order], axis=1) elif self.use_deep: concat_input = self.y_deep self.out = tf.add( tf.matmul(concat_input, self.weights["concat_projection"]), self.weights["concat_bias"]) # loss if self.loss_type == "logloss": self.out = tf.nn.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == "mse": self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # l2 regularization on weights if self.l2_reg > 0: # todo 这里FM的正则只加FM之后全连接的正则,没有加前面FM中的正则 # regularizer = tf.contrib.layers.l2_regularizer(scale=0.1) #scale代表正则化系数的值 # tf中通过tf.contrib.layers.l2_regularizer(scale=0.1) 创建一个正则化方法,此处是L2正则化,#scale代表正则化系数的值。 self.loss += tf.contrib.layers.l2_regularizer(self.l2_reg)( self.weights["concat_projection"]) if self.use_deep: for i in range(len(self.deep_layers)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["layer_%d" % i]) # optimizer if self.optimizer_type == "adam": self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss) elif self.optimizer_type == "adagrad": self.optimizer = tf.train.AdagradOptimizer( learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize(self.loss) elif self.optimizer_type == "gd": self.optimizer = tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate).minimize(self.loss) elif self.optimizer_type == "momentum": self.optimizer = tf.train.MomentumOptimizer( learning_rate=self.learning_rate, momentum=0.95).minimize(self.loss) elif self.optimizer_type == "yellowfin": pass self.optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize(self.loss) # init # 这里建了saver没用到吧 self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = self._init_session() self.sess.run(init) # number of params # todo 对模型训练没什么帮助 total_parameters = 0 for variable in self.weights.values(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters if self.verbose > 0: print("#params: %d" % total_parameters)