def __init__(self,
n_actions,
state_size,
architecture,
epsilon_exp_decay=0.999,
epsilon_final=0.05,
gamma=0.99,
exp_replay_size=int(1e5),
batch_size=128,
initial_exploration_steps=1e4,
tau=1e-3,
learning_rate=1e-3,
update_every=1):
self.epsilon = 1.0
self.epsilon_final = epsilon_final
self.eps_decay = epsilon_exp_decay
self.n_actions = n_actions
self.state_size = state_size
self.gamma = gamma
self.exp_replay = ExperienceReplay(size=exp_replay_size)
self.batch_size = batch_size
self.neural_net = NNModel(arch=architecture,
batch_size=self.batch_size,
n_outputs=n_actions,
state_shape=state_size,
learning_rate=learning_rate)
self.target_net = NNModel(arch=architecture,
batch_size=self.batch_size,
n_outputs=n_actions,
state_shape=state_size,
learning_rate=learning_rate)
self.initial_exploration_steps = initial_exploration_steps
self.tau = tau
self.update_every = update_every
self.c = 0
def _train_model(self, config, **kwargs):
trainer = NNTrainer(
self.feature_extractor,
num_epochs=config["epochs"],
balance_method=config["balance_method"],
class_weight_method=config["class_weight_method"],
)
model = NNModel(
self.nn_class,
self.feature_extractor.feature_size,
dropout=config["dropout"],
)
val_acc, val_loss = trainer.train(model)
trainer.evaluate(model, trainer.feature_dataset.validation_loader)
return val_acc, val_loss, model
def train(args):
test, train, val = datasets.get_partitions(args)
model = NNModel(args)
# insert other definitions here, e.g. callbacks
# calls the fit function for the model class; actual calls to TF and sklearn happen in the class function
model.fit(data=train, val_data=val) # other parameters as well
model.evaluate(data=test)
from models import NNModel
from models.pretrained_model import PretrainedNNTrainer
feature_extractor = ResNetCustom("./models/grid_search_resnet_custom/best.pth")
print("Loading CNN Model")
cnn_model = models.PretrainedNNModel(
tv_models.resnet152,
transfers.final_layer_alteration_resnet,
state_dict_path="./models/grid_search_resnet_custom/best.pth",
eval_mode=True,
)
print("Loading Linear Model")
lnn_model = NNModel(models.LinearNN,
feature_extractor.feature_size,
eval_mode=True)
print("Setting fc weights")
lnn_model.net.fc1.weight = cnn_model.net.fc.weight
lnn_model.net.fc1.bias = cnn_model.net.fc.bias
print("Evaluating CNN:")
image_datasets = ImageDatasets()
PretrainedNNTrainer.evaluate(
cnn_model,
image_datasets.get_loader(DatasetType.Train),
apply_softmax=True,
verbose=True,
)
if __name__ == "__main__":
import pickle as pkl
dr = YahooDataReader(None)
dr.data = pkl.load(open("GermanCredit/german_train_rank.pkl", "rb"))
vdr = YahooDataReader(None)
vdr.data = pkl.load(open("GermanCredit/german_test_rank.pkl", "rb"))
args = parse_my_args_reinforce()
torch.set_num_threads(args.num_cores)
args.group_feat_id = 3
if args.model_type == "Linear":
model = LinearModel(D=args.input_dim, clamp=args.clamp)
print("Linear model initialized")
else:
model = NNModel(D=args.input_dim,
hidden_layer=args.hidden_layer,
dropout=args.dropout,
pooling=args.pooling,
clamp=args.clamp)
print(
"Model initialized with {} hidden layer size, Dropout={}, using {} pooling"
.format(args.hidden_layer, args.dropout, args.pooling))
model = demographic_parity_train(model, dr, vdr, vvector(200), args)
results = evaluate_model(model,
vdr,
fairness_evaluation=False,
group_fairness_evaluation=True,
deterministic=True,
args=args,
num_sample_per_query=100)
else:
loss_function = self.loss()
# Setup trial
trial = Trial(net, optimiser, loss_function, metrics=["loss", "accuracy"]).to(
device
)
trial.with_generators(
train_loader, test_generator=validation_loader,
)
# Actually run the training
trial.run(epochs=self.num_epochs)
# Evaluate and show results
time.sleep(0.1) # Ensure training has finished
net.eval()
results = trial.evaluate(data_key=torchbearer.TEST_DATA)
acc = float(results["test_acc"])
loss = float(results["test_loss"])
return acc, loss
if __name__ == "__main__":
_network_class = models.BiggerNN
_feature_extractor = features.AlexNet()
_trainer = NNTrainer(_feature_extractor, balance_method=BalanceMethod.OverSample)
_model = NNModel(_network_class, _feature_extractor.feature_size)
_trainer.train(_model)
class QAgent():
def __init__(self,
n_actions,
state_size,
architecture,
epsilon_exp_decay=0.999,
epsilon_final=0.05,
gamma=0.99,
exp_replay_size=int(1e5),
batch_size=128,
initial_exploration_steps=1e4,
tau=1e-3,
learning_rate=1e-3,
update_every=1):
self.epsilon = 1.0
self.epsilon_final = epsilon_final
self.eps_decay = epsilon_exp_decay
self.n_actions = n_actions
self.state_size = state_size
self.gamma = gamma
self.exp_replay = ExperienceReplay(size=exp_replay_size)
self.batch_size = batch_size
self.neural_net = NNModel(arch=architecture,
batch_size=self.batch_size,
n_outputs=n_actions,
state_shape=state_size,
learning_rate=learning_rate)
self.target_net = NNModel(arch=architecture,
batch_size=self.batch_size,
n_outputs=n_actions,
state_shape=state_size,
learning_rate=learning_rate)
self.initial_exploration_steps = initial_exploration_steps
self.tau = tau
self.update_every = update_every
self.c = 0
def choose_action(self, state, greedy=False):
q_values = self._get_q_values(state, use_target_net=False)
best_action = np.argmax(q_values)
if greedy:
# Choose the greedy action
action = best_action
else:
# Perform epsilon-greedy and update epsilon
if random.random() > self.epsilon:
action = best_action
else:
action = random.choice(range(self.n_actions))
return action
def _update_epsilon(self):
# Epsilon exponential decay
self.epsilon = self.eps_decay * self.epsilon + (
1 - self.eps_decay) * self.epsilon_final
def _get_q_values(self, state, use_target_net=True):
if len(state.shape) <= len(self.state_size):
state = np.expand_dims(state, 0)
if use_target_net:
q_values = self.target_net.predict(state)
else:
q_values = self.neural_net.predict(state)
assert q_values.shape[1] == self.n_actions
assert len(q_values.shape) == 2
return q_values
def step(self, state, action, reward, next_state):
self.c += 1
# 1. Add observation to the deque
observation = (np.squeeze(state), action, reward,
np.squeeze(next_state))
self.exp_replay.append(observation)
# 2. Update the neural net
if (self.exp_replay.length > self.initial_exploration_steps):
self._update_epsilon()
if ((self.c % self.update_every) == 0):
self.c += 1
sample = self.exp_replay.draw_sample(
sample_size=self.batch_size)
states, actions, rewards, next_states = sample
# Train
batch_x = states
batch_y = rewards + self.gamma * np.max(
self._get_q_values(next_states, use_target_net=True),
axis=1,
keepdims=True) # TD_target
self.neural_net.train(batch_x, batch_y, actions)
# update target net
self.target_net.copy_weights_from(self.neural_net.net,
tau=self.tau)