batch_size = 1400
initial_learning_rate = 0.001
keep_probs = None
max_lbfgs_iter = 1000
decay_epochs = [1000, 10000]
tf.reset_default_graph()
tf_model = LogisticRegressionWithLBFGS(
input_dim=input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=data_sets,
initial_learning_rate=initial_learning_rate,
# damping=1e-2, # try changing? 80
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name='fashion_mnist_small_logreg_lbfgs')
tf_model.train()
X_train = np.copy(tf_model.data_sets.train.x)
print('X_train.shape', X_train.shape)
Y_train = np.copy(tf_model.data_sets.train.labels)
X_test = np.copy(tf_model.data_sets.test.x)
Y_test = np.copy(tf_model.data_sets.test.labels)
weight_decay = 0.01
batch_size = 1400
initial_learning_rate = 0.001
keep_probs = None
max_lbfgs_iter = 1000
decay_epochs = [1000, 10000]
tf.reset_default_graph()
tf_model = LogisticRegressionWithLBFGS(
input_dim=input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=data_sets,
initial_learning_rate=initial_learning_rate,
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name='mnist_logreg_lbfgs')
tf_model.train()
test_idx = 8
actual_loss_diffs, predicted_loss_diffs_cg, indices_to_remove = experiments.test_retraining(
tf_model,
test_idx,
iter_to_load=0,
force_refresh=False,
def init_model(self):
"""
Initialize a tf model based on model_name and datasets
"""
# TODO: make it easier to use non-default hyperparams?
# we can always infer # classes of from the training data
num_classes = len(set(self.data_sets.train.labels))
model_name = self.task + '_' + self.model_name
print('Num classes', num_classes)
if self.model_name == 'binary_logistic':
#num_classes = 2
assert num_classes == 2
weight_decay = 0.0001
batch_size = 100
initial_learning_rate = 0.001
keep_probs = None
decay_epochs = [1000, 10000]
max_lbfgs_iter = 1000
self.model = BinaryLogisticRegressionWithLBFGS(
input_dim=self.input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=self.data_sets,
initial_learning_rate=initial_learning_rate,
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name=model_name
)
elif self.model_name == 'multi_logistic':
#num_classes = 10
weight_decay = 0.01
batch_size = 1400
initial_learning_rate = 0.001
keep_probs = None
max_lbfgs_iter = 1000
decay_epochs = [1000, 10000]
self.model = LogisticRegressionWithLBFGS(
input_dim=self.input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=self.data_sets,
initial_learning_rate=initial_learning_rate,
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name=model_name)
elif self.model_name == 'cnn':
assert num_classes == 10
weight_decay = 0.001
batch_size = 500
initial_learning_rate = 0.0001
decay_epochs = [10000, 20000]
hidden1_units = 8
hidden2_units = 8
hidden3_units = 8
conv_patch_size = 3
keep_probs = [1.0, 1.0]
self.model = All_CNN_C(
input_side=self.input_side,
input_channels=self.input_channels,
conv_patch_size=conv_patch_size,
hidden1_units=hidden1_units,
hidden2_units=hidden2_units,
hidden3_units=hidden3_units,
weight_decay=weight_decay,
num_classes=num_classes,
batch_size=batch_size,
data_sets=self.data_sets,
initial_learning_rate=initial_learning_rate,
damping=1e-2,
decay_epochs=decay_epochs,
mini_batch=True,
train_dir='output',
log_dir='log',
model_name=model_name
)
elif self.task == 'income':
num_classes = 2
input_dim = self.data_sets.train.x.shape[1]
weight_decay = 0.0001
# weight_decay = 1000 / len(lr_data_sets.train.labels)
batch_size = 10
initial_learning_rate = 0.001
keep_probs = None
decay_epochs = [1000, 10000]
max_lbfgs_iter = 1000
self.model = BinaryLogisticRegressionWithLBFGS(
input_dim=input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=self.data_sets,
initial_learning_rate=initial_learning_rate,
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name='income_logreg'
)
elif self.model_name == 'hinge_svm':
#num_classes = 2
weight_decay = 0.01
use_bias = False
batch_size = 100
initial_learning_rate = 0.001
keep_probs = None
decay_epochs = [1000, 10000]
temps = [0, 0.001, 0.1]
num_temps = len(temps)
num_params = 784
temp = 0
self.model = SmoothHinge(
use_bias=use_bias,
temp=temp,
input_dim=self.input_dim,
weight_decay=weight_decay,
num_classes=num_classes,
batch_size=batch_size,
data_sets=self.data_sets,
initial_learning_rate=initial_learning_rate,
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name='smooth_hinge_17_t-%s' % temp)
batch_size = 1400
initial_learning_rate = 0.001
keep_probs = None
max_lbfgs_iter = 1000
decay_epochs = [1000, 10000]
tf.reset_default_graph()
tf_model = LogisticRegressionWithLBFGS(
input_dim=input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=data_sets,
initial_learning_rate=initial_learning_rate,
# damping=1e-2, # try changing? 80
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name='fashion_mnist_small_logreg_lbfgs')
tf_model.train()
X_train = np.copy(tf_model.data_sets.train.x)
print('X_train.shape', X_train.shape)
Y_train = np.copy(tf_model.data_sets.train.labels)
X_test = np.copy(tf_model.data_sets.test.x)
Y_test = np.copy(tf_model.data_sets.test.labels)
batch_size = 1400
initial_learning_rate = 0.001
keep_probs = None
max_lbfgs_iter = 1000
decay_epochs = [1000, 10000]
tf.reset_default_graph()
tf_model = LogisticRegressionWithLBFGS(
input_dim=input_dim,
weight_decay=weight_decay,
max_lbfgs_iter=max_lbfgs_iter,
num_classes=num_classes,
batch_size=batch_size,
data_sets=data_sets,
initial_learning_rate=initial_learning_rate,
# damping=1e-2, # try changing? 80
keep_probs=keep_probs,
decay_epochs=decay_epochs,
mini_batch=False,
train_dir='output',
log_dir='log',
model_name='fashion_mnist_small_logreg_lbfgs')
tf_model.train()
X_train = np.copy(tf_model.data_sets.train.x)
print('X_train.shape', X_train.shape)
Y_train = np.copy(tf_model.data_sets.train.labels)
X_test = np.copy(tf_model.data_sets.test.x)
Y_test = np.copy(tf_model.data_sets.test.labels)