def __init__(self, evaluator, evaluable, **args):
""" The evaluator must be an episodic task, an the evaluable must be a module. """
assert isinstance(evaluator, EpisodicTask)
assert isinstance(evaluable, Module)
self.agent = self.subagent(evaluable.copy(), self.sublearner(**self.subargs))
self.agent.learner.setAlpha(self.learningRate)
self.agent.learner.gd.momentum = self.momentum
self.agent.copy = lambda: self.agent
self.module = evaluable.copy()
def wrappedEvaluator(module):
""" evaluate the internal agent (and changing its internal parameters),
and then transferring the parameters to the outside basenet """
self.agent.reset()
res = 0.
for dummy in range(self.learningBatches):
res += evaluator(self.agent)
res /= self.learningBatches
self.agent.learn()
module._setParameters(self.agent.module.params[:module.paramdim])
# the performance is measured on a greedy run:
res2 = evaluator(module)
if self.verbose:
print 'stoch', res, 'greedy', res2,
return res2
Learner.__init__(self, wrappedEvaluator, evaluable, **args)
def __init__(self, args, config):
"""
:param args:
"""
super(Meta, self).__init__()
# self.update_lr = args.update_lr
self.meta_lr = args.meta_lr
self.n_way = args.n_way
self.k_spt = args.k_spt
self.k_qry = args.k_qry
self.task_num = args.task_num
self.update_step = args.update_step
self.update_step_test = args.update_step_test
self.net = Learner(config, args.imgc, args.imgsz)
self.update_lrs = nn.ParameterList([
nn.Parameter(args.update_lr *
torch.ones(self.update_step, var.size(0)),
requires_grad=True) for var in self.net.vars
])
self.meta_optim = optim.Adam(self.parameters(), lr=self.meta_lr)
self.lr_scheduler = optim.lr_scheduler.ExponentialLR(
self.meta_optim, 0.9)
def __init__(self, meta, layers=[], rate=.05, target=None, momentum=None, trans=None, wrange=100):
Learner.__init__(self, meta, target)
inputs = len(self.meta.names()) - 1
_, possible = self.meta[self.target]
self.outputs = possible
self.net = Net([inputs] + layers + [len(possible)], rate=rate, momentum=momentum, wrange=wrange, trans=trans)
def main():
dbinfo = recover()
conn = MySQLdb.connect(**dbinfo)
cur = conn.cursor()
#Learn
sql = "SELECT id,article_text,trainpos,trainneg,trainneutral FROM articles WHERE trainset=1 AND (trainpos>0 OR trainneg>0 OR trainneutral>0)"
cur.execute(sql)
a = Learner()
for aid,article_text,trainpos,trainneg,trainneutral in cur.fetchall():
aid = int(aid)
items = [ (1, int(trainpos)),(0, int(trainneutral)),(-1, int(trainneg)) ]
classification = max(items, key=lambda x : x[1])[0]
a.add_string(article_text, classification)
a.train()
#Predict
sql = "SELECT id,article_text FROM articles"
cur.execute(sql)
b = Classifier(a)
for aid,article_text in cur.fetchall():
aid = int(aid)
classification = b.classify(article_text)
sql = "UPDATE articles SET score=%s WHERE id=%s"
args = [classification,aid]
cur.execute(sql,args)
print aid,classification
conn.commit()
class LearnerTestCase(ut.TestCase): def setUp(self): self.board = Board() self.learner = Learner() def tearDown(self): self.board = None self.learner = None def testMinimax(self): # self.board= Board(new_grid = RED_EASY_LOOKAHEAD) # print(self.learner.getNextMove(self.board)) # self.board= Board(new_grid = START) self.board = Board(new_grid = RED_EASY_LOOKAHEAD_2) best = self.learner.getNextMove(self.board) def testNearestNeighbor(self): weights = [0] * len(self.board.getMoveList(AI_COLOR)) weights[0] = 1 self.learner = Learner(data_points = [(self.board.getArray().tolist(), weights)]) # self.board.getMoveList(AI_COLOR)[0][1].printMove() # self.learner.getNextMove(self.board).printBoard() self.assertEqual(self.learner.getNextMove(self.board), self.board.getMoveList(AI_COLOR)[0][1], \ 'predicted best move does not match') def testUpdateWeights(self): pass
def __init__(self, args, config):
"""
:param args:
"""
super(Meta, self).__init__()
# self.update_lr = args.update_lr
self.meta_lr = args.meta_lr
self.n_way = args.n_way
self.k_spt = args.k_spt
self.k_qry = args.k_qry
self.task_num = args.task_num
self.update_step = args.update_step
self.update_step_test = args.update_step_test
self.reg_lambda = args.reg_lambda
self.regularization = nn.MSELoss()
self.net = Learner(config, args.imgc, args.imgsz)
self.update_lrs = nn.Parameter(
args.update_lr * torch.ones(self.update_step, len(self.net.vars)),
requires_grad=True)
self.meta_optim = optim.Adam(self.parameters(), lr=self.meta_lr)
#self.CG_optim = HessianFree(self.parameters(), lr=self.meta_lr, cg_max_iter=5)
self.lr_scheduler = optim.lr_scheduler.ExponentialLR(
self.meta_optim, args.exp_decay)
self.MAX_CG = args.MAX_CG
def __init__(self):
self.learn = learn
self.load = load
self.learner = Learner()
if self.load and os.path.isfile(learnerFile):
self.learner.loadData(learnerFile)
self.learner.newGame()
def main():
printer = Printer('log.txt')
utils = Utils(printer)
utils.setup_and_verify()
utils.evaluate_baseline()
learner = Learner(utils.learner_utils)
learner.train_and_evaluate()
utils.printer.print('finished!')
def __init__(self, meta, rate, target=None):
Learner.__init__(self, meta, target)
length = len(meta.names())
_, possible = meta[self.target]
self.perceptrons = {}
for truthy, falsy in itertools.combinations(possible, 2):
self.perceptrons[(truthy, falsy)] = Perceptron(length, rate)
def __init__(self, name, selection_strategy):
"""
Initializes a new incremental learner
:param str name: The learner name
:param SelectionStrategy selection_strategy: The selection strategy
"""
Learner.__init__(self, "incremental_{}".format(name))
self.selection_strategy = selection_strategy
self.observer = observe.DispatchObserver()
def createLearner(self):
learner = Learner(self.port + 2, self.ips, self.ip, self.num)
if not self.logging_switch:
learner.logging(False)
learner.log('starting')
learner.listen()
learner.log('exiting')
def train(conf):
gan = KernelGAN(conf)
learner = Learner()
data = DataGenerator(conf, gan)
for iteration in tqdm.tqdm(range(conf.max_iters), ncols=60):
[g_in, d_in] = data.__getitem__(iteration)
gan.train(g_in, d_in)
learner.update(iteration, gan)
gan.finish()
def __init__(self, data, init_model, param): Learner.__init__(self, data, init_model, param) # print 'Hey, TheanoLearner is initializing!' ### mapping from input data to continuous integer indices # label_set['label'] = j self.label_map = get_label_map(self.labels) # feature_set['fea_id'] = k self.feature_map = get_feature_map(data.dat)
def _initActors(self):
self.proposer = Proposer(self.servers, self.id)
self.acceptor = Acceptor(self.servers, self.id)
self.learner = Learner(self.servers, self.id)
# load saved
if os.path.isfile(self.stateFileName):
print('Loading from:', self.stateFileName)
self._loadState()
def __init__(self):
super().__init__()
# self.gv = gv
self.learner = Learner()
self.pg_layout = self.create_tab()
self.tool_dock = self.create_settings_dock()
self.create_open_settings_button()
result_dock, self.training_plot = self.create_result_dock()
self.create_test_unseen_data_dock(result_dock)
self.lr = 0
def estimate_kernel(img_file):
conf = config_kernelGAN(img_file)
kgan = KernelGAN(conf)
learner = Learner()
data = DataGenerator(conf, kgan)
for iteration in tqdm.tqdm(range(conf.max_iters), ncols=70):
[g_in, d_in, _] = data.__getitem__(iteration)
kgan.train(g_in, d_in)
learner.update(iteration, kgan)
kgan.finish()
def train(conf):
sr_net = DBPISR(conf)
learner = Learner()
data = DataGenerator(conf, sr_net)
for iteration in tqdm.tqdm(range(conf.max_iters), ncols=60):
g_in = data.__getitem__(iteration)
sr_net.train(g_in)
learner.update(iteration, sr_net)
sr_net.finish(data.input_image)
def __init__(self,
id,
quorum_size,
is_leader=False,
promised_id=None,
accepted_id=None,
accepted_value=None):
Proposer.__init__(self, id, quorum_size, is_leader)
Acceptor.__init__(self, id, promised_id, accepted_id, accepted_value)
Learner.__init__(self, id, quorum_size)
def playLoser(winner, game_count):
loser = Learner()
wins = 0
losses = 0
ties = 0
for game in tqdm(range(game_count)):
game_board = Board()
winner_moves = []
loser_moves = []
game_history = []
turn_count = 0
tie_flag = False
while game_board.checkGameStatus(AI_COLOR) == CONTINUE:
if turn_count > 100:
tie_flag = True
break
loser_move = loser.getNextMove(game_board.getInverse())
loser_moves.append(loser_move.getInverse())
assert(loser_move is not None and game_board is not None)
game_history.append((game_board, loser_move.getInverse()))
game_board = Board(game_board.applyMove(loser_move.getInverse()))
if game_board.checkGameStatus(AI_COLOR) == LOSE:
break
# assert (temp != game_board)
winner_move = winner.getNextMove(game_board)
assert(winner_move is not None and game_board is not None)
game_history.append((game_board, winner_move))
winner_moves.append(winner_move)
game_board = Board(game_board.applyMove(winner_move))
turn_count += 1
if tie_flag or game_board.checkGameStatus(AI_COLOR) == TIE:
ties += 1
elif game_board.checkGameStatus(AI_COLOR) == WIN:
wins += 1
elif game_board.checkGameStatus(AI_COLOR) == LOSE:
losses += 1
return { WIN : wins, LOSE : losses, TIE : ties}
def __init__(self, cfg):
self.cfg = cfg
self.tbwriter = TensorBoardWriter(cfg)
self.replay_buffer = ReplayBuffer(cfg)
self.learner_outqueue = queue.Queue()
self.learner = Learner(cfg, replay_buffer=self.replay_buffer, outqueue=self.learner_outqueue)
self.learner_thread = LearnerThread(self.learner)
self.learner_thread.start()
self.learner_post_thread = LearnerPostThread(cfg, self.learner_outqueue, self.tbwriter)
self.learner_post_thread.start()
def train_and_eval(conf):
model = DualSR(conf)
dataloader = create_dataset(conf)
learner = Learner(model)
print('*' * 60 + '\nTraining started ...')
for iteration, data in enumerate(tqdm.tqdm(dataloader)):
model.train(data)
learner.update(iteration, model)
model.eval()
def __init__(self, args, config):
super(Meta, self).__init__()
self.update_lr = args.update_lr
self.meta_lr = args.meta_lr
self.n_way = args.n_way
self.task_num = args.task_num
self.update_step = args.update_step
self.net = Learner(config)
self.meta_optim = optim.Adam(self.net.parameters(), lr=self.meta_lr)
def signature_dendrogram(flows: [{}]):
pc = Analyzer.url_clustering(flows)
pc = sorted(pc)
dm = np.asarray([[Analyzer.url_pattern_dist(p1, p2) for p2 in pc]
for p1 in pc])
dm = sparse.csr_matrix(dm)
dm = ssd.squareform(dm.todense())
Z = linkage(dm)
Learner.fancy_dendrogram(Z, leaf_rotation=90., leaf_font_size=8)
for i in range(len(pc)):
logger.info('Signature %d: %s', i, pc[i])
return Z, pc
def __init__(self, args, config, task_mod):
"""
:param args:
"""
super(AL_Learner, self).__init__()
self.meta_lr = args.meta_lr
self.k_qry = args.k_qry
self.task_num = args.task_num
self.task_net = task_mod
self.net = Learner(config, args.imgc, args.imgsz)
self.meta_optim = optim.Adam(self.net.parameters(), lr=self.meta_lr)
def train(instances, contexts_dir, resource='Location'):
logger.info('The number of instances: %d', len(instances))
# Convert the text_fea into the <String, label> pairs.
docs, y = ContextProcessor.docs(instances, ['r_' + resource])
# Transform the strings into the np array.
train_data, voc, vec = Learner.LabelledDocs.vectorize(docs)
logger.info('neg: %d', len(np.where(y == 0)[0]))
logger.info('pos: %d', len(np.where(y == 1)[0]))
# Split the data set into 10 folds.
folds = Learner.n_folds(
train_data, y, fold=10
) # [Fold(f) for f in Learner.n_folds(train_data, y, fold=10)]
# Wrap a bunch of classifiers and let them vote on every fold.
clfs = [
svm.SVC(kernel='linear', class_weight='balanced',
probability=True),
RandomForestClassifier(class_weight='balanced'),
LogisticRegression(class_weight='balanced')
]
res = Learner.voting(clfs, train_data, y, folds)
for clf in clfs:
clf_name = type(clf).__name__
logger.debug('CLF: %s', clf_name)
for fold_id in res[clf_name]['folds']:
fold = res[clf_name]['folds'][fold_id]
if 'fp_item' not in fold:
continue
for fp in fold['fp_item']:
logger.debug('FP: %s, %s, %s', str(instances[fp].ui_doc),
instances[fp].topic, instances[fp].app_name)
for fn in fold['fn_item']:
logger.debug('FN: %s, %s, %s', str(instances[fn].ui_doc),
instances[fn].topic, instances[fn].app_name)
with open(os.path.join(contexts_dir, 'folds.json'), 'w') as json_file:
for fold_id in folds:
fold = folds[fold_id]
fold['train_index'] = fold['train_index'].tolist()
fold['test_index'] = fold['test_index'].tolist()
# pd.Series(folds).to_json(json_file, orient='values')
logger.info('The number of folds: %d', len(folds))
json.dump(folds, json_file)
with open(os.path.join(contexts_dir, 'voting_res.json'),
'w') as json_file:
pd.Series(res).to_json(json_file, orient='split')
logger.info(
'The total number of overlapping instances after voting: %d',
len(res['voting']['all']))
logger.info('The number of fp: %d', len(res['voting']['fp']))
logger.info('The number of fn: %d', len(res['voting']['fn']))
logger.info('conf_mat: %s', str(res['voting']['conf_mat']))
def test():
df = pd.DataFrame({
"area": [2104, 1416, 1534, 852, 124],
"#bedrooms": [5, 3, 3, 2, 1],
"#floors": [1, 2, 2, 1, 1],
"age": [45, 40, 30, 36, 20],
})
targets = pd.Series([460, 232, 315, 178, 130])
learner = Learner(df, targets, linear_regression.hypo,
linear_regression.loss, linear_regression.row_gradient)
print(learner.fit(100, 0.0000001))
def __init__(self, args, abnorm_feats, norm_feats, metric, chpt=None):
"""
:param args:
"""
super(Meta, self).__init__()
# meta train config
self.inner_lr = args.inner_lr
self.meta_lr = args.meta_lr
self.n_way = args.n_way
self.num_task = args.num_task
self.batch_size = args.batch_size
self.kshot = args.kshot
self.epochs = args.inner_epoch
self.w_decay = args.w_decay
self.grad_clip = args.grad_clip
self.inner_optim = 'sgd'
# when inner_optim == 'adam'
self.eps = 1e-8
self.beta1 = 0.9
self.beta2 = 0.999
self.device = torch.device('cuda')
self.net = Learner(args.feat_dim, drop_p=0).to(self.device)
if chpt is not None:
print('Load pretrained model!')
self.net.load_state_dict(torch.load(chpt))
self.loss_fn = custom_objective
self.abnorm_feats = abnorm_feats
self.norm_feats = norm_feats
self.metric = metric
self.meta_sgd = args.meta_sgd
if self.meta_sgd:
self.lr_params = self.get_lr_params()
self.learner_optim = optim.Adam([{
'params': self.net.parameters(),
'lr': args.meta_lr,
'weight_decay': args.w_decay
}, {
'params': self.lr_params,
'lr': args.lr_lr
}])
else:
self.learner_optim = optim.Adam(self.net.parameters(),
lr=self.meta_lr,
weight_decay=args.w_decay)
def __init__(self, args, config):
super(Meta, self).__init__()
self.update_lr = args.update_lr
self.meta_lr = args.meta_lr
self.n_way = args.n_way
self.k_spt = args.k_spt
self.k_qry = args.k_qry
self.task_num = args.task_num
self.update_step = args.update_step
self.update_step_test = args.update_step_test
self.meta_optim_choose = args.meta_optim_choose
self.net = Learner(config, args.imgc, args.imgsz)
self.meta_optim = optim.Adam(self.net.parameters(), lr=self.meta_lr)
def __init__(self,
start,
goal,
Xrange,
Vrange,
num_actions=3,
max_memory=500,
hidden_size=200,
learning_rate=.001,
discount_factor=.99,
epsilon=.1):
Learner.__init__(self, start, goal, Xrange, Vrange, num_actions,
max_memory, hidden_size, learning_rate,
discount_factor, epsilon)
self.model1, self.model2 = self.build_model(), self.build_model()
def __init__(self, args, config):
super(Meta, self).__init__()
self.update_lr = args.update_lr
self.meta_lr = args.meta_lr
self.n_way = args.n_way
self.k_shot = args.k_shot
self.k_query = args.k_query
self.task_num = args.task_num
self.update_step = args.update_step
self.update_step_test = args.update_step_test
self.net = Learner(config) ## base-learner
self.meta_optim = torch.optim.Adam(self.net.parameters(),
lr=self.meta_lr)
def gen_docs(jsons: [{}],
char_wb: bool = False,
add_taint: bool = False) -> [Learner.LabelledDocs]:
"""
Generate string list from the flow URLs.
:param jsons: The flow jsons.
:param char_wb:
:param add_taint: Whether add taints as tokens.
:return:
"""
docs = []
taint_counts = 0
for flow in jsons:
line = Analyzer.filter_url_words(flow['url'])
if '_' in flow['taint']:
taint_counts += 1
if add_taint:
line = line + ' ' + 't_' + flow['taint']
label = 1 if flow['label'] == '1' else 0
real_label = 1 if flow['real_label'] == '1' else 0
if real_label != label:
logger.info(
"Flow's real label does not match the training label for %s, real_label = %d label = %d",
flow['url'], real_label, label)
numeric = [flow[name] for name in Analyzer.numeric_features]
docs.append(
Learner.LabelledDocs(line,
label,
numeric,
real_label,
char_wb=char_wb))
logger.info('The number of flows who have more than 1 taints: %d',
taint_counts)
return docs
def train(conf):
gan = KernelGAN(conf)
learner = Learner()
data = DataGenerator(conf, gan)
dataloader = DataLoader(data, batch_size=batch_size,
shuffle=False)
timer = 0
for i_batch, sample_batched in enumerate(tqdm.tqdm(dataloader)):
g_in,d_in = sample_batched
gan.train(g_in,d_in)
learner.update(i_batch*batch_size, gan)
if learner.flag:
timer += 1
if timer > 10:
break
gan.finish()
def setup_master(self):
policy1 = None
policy2 = None
team1 = [
] ## List of Neo objects, one Neo object per agent, teams_list = [team1, teams2]
team2 = []
num_adversary = 0
num_friendly = 0
for i, agent in enumerate(self.world.policy_agents):
if agent.attacker:
num_adversary += 1
else:
num_friendly += 1
action_space = self.action_spaces[
i] ##* why on earth would you put i???
pol_obs_dim = self.observation_spaces[0].shape[
0] ##* ensure 27 is correct
# index at which agent's position is present in its observation
pos_index = 2 ##* don't know why it's a const and +2
for i, agent in enumerate(self.world.policy_agents):
obs_dim = self.observation_spaces[i].shape[0]
if not agent.attacker: # first we have the guards and then the attackers
if policy1 is None:
policy1 = MPNN(input_size=pol_obs_dim,
num_agents=num_friendly,
num_opp_agents=num_adversary,
num_entities=0,
action_space=self.action_spaces[i],
pos_index=pos_index,
mask_dist=1.0,
entity_mp=False,
policy_layers=1).to(self.device)
team1.append(
Neo(self.namespace_args, policy1, (obs_dim, ),
action_space)
) ## Neo adds additional features to policy such as loading model, update_rollout. Neo.actor_critic is the policy and is the same object instance within a team
else:
if policy2 is None:
policy2 = MPNN(input_size=pol_obs_dim,
num_agents=num_adversary,
num_opp_agents=num_friendly,
num_entities=0,
action_space=self.action_spaces[i],
pos_index=pos_index,
mask_dist=1.0,
entity_mp=False).to(self.device)
team2.append(
Neo(self.namespace_args, policy2, (obs_dim, ),
action_space))
master = Learner(self.namespace_args, [team1, team2],
[policy1, policy2],
world=self.world)
return master
def main(config):
# Select Model
model = Model.factory(config)
# Build Train Edges
data = TrainData(config.graph_path)
# Build Train Data
train_iter = GraphGenerator(
graph_wrappers=model.graph_wrappers,
batch_size=config.batch_size,
data=data,
samples=config.samples,
num_workers=config.sample_workers,
feed_name_list=[var.name for var in model.feed_list],
use_pyreader=config.use_pyreader,
phase="train",
graph_data_path=config.graph_path,
shuffle=True)
log.info("build graph reader done.")
learner = Learner.factory(config.learner_type)
learner.build(model, train_iter, config)
learner.start()
learner.stop()
def main():
# Load individual events
with open('training_data/events.json', 'r') as infile:
training_data = json.load(infile)
with open('test_data/events.json', 'r') as infile:
test_data = json.load(infile)
# Classify events
learner = Learner()
learner.train(training_data[0], training_data[1])
test_labels = [learner.classify(i) for i in list(test_data)]
test_labels = [str(i) for i in test_labels]
with open('test_data/labels.json', 'w') as outfile:
json.dump(list(test_labels), outfile, indent=4)
def evaluate(f, optimizer, best_sum_loss, best_final_loss, best_flag, lr):
print('\n --> evalute the model')
STEPS = 100
LSTM_learner = Learner(f,
optimizer,
STEPS,
eval_flag=True,
reset_theta=True,
retain_graph_flag=True)
lstm_losses, sum_loss = LSTM_learner()
try:
best = torch.load('best_loss.txt')
except IOError:
print('can not find best_loss.txt')
pass
else:
best_sum_loss = best[0]
best_final_loss = best[1]
print("load_best_final_loss and sum_loss")
if lstm_losses[-1] < best_final_loss and sum_loss < best_sum_loss:
best_final_loss = lstm_losses[-1]
best_sum_loss = sum_loss
print(
'\n\n===> best of final LOSS[{}]: = {}, best_sum_loss ={}'.format(
STEPS, best_final_loss, best_sum_loss))
torch.save(optimizer.state_dict(), 'best_LSTM_optimizer.pth')
torch.save([best_sum_loss, best_final_loss, lr], 'best_loss.txt')
best_flag = True
return best_sum_loss, best_final_loss, best_flag
def main(config, max_samples):
get_env_configs(config)
ray.init()
parameter_server = ParameterServer.remote(config)
replay_buffer = ReplayBuffer.remote(config)
learner = Learner.remote(config, replay_buffer, parameter_server)
train_actor_ids = []
eval_actor_ids = []
learner.start_learning.remote()
# start train actors
for i in range(config["num_workers"]):
epsilon = config["max_eps"] * i / config["num_workers"]
training_actor = Actor.remote("train-" + str(i), replay_buffer,
parameter_server, config, epsilon)
training_actor.sample.remote()
train_actor_ids.append(training_actor)
# start eval actors
for i in range(config["eval_num_workers"]):
epsilon = 0
eval_actor = Actor.remote("eval-" + str(i),
replay_buffer,
parameter_server,
config,
epsilon,
eval=True)
eval_actor_ids.append(eval_actor)
# fetch samples in loop and sync actor weights
total_samples = 0
best_eval_mean_reward = np.NINF
eval_mean_rewards = []
while total_samples < max_samples:
total_env_samples_id = replay_buffer.get_total_env_samples.remote()
new_total_samples = ray.get(total_env_samples_id)
num_new_samples = new_total_samples - total_samples
if num_new_samples >= config["timesteps_per_iteration"]:
total_samples = new_total_samples
print("Total samples:", total_samples)
parameter_server.set_eval_weights.remote()
eval_sampling_ids = [
eval_actor.sample.remote() for eval_actor in eval_actor_ids
]
eval_rewards = ray.get(eval_sampling_ids)
print("Evaluation rewards: {}".format(eval_rewards))
eval_mean_reward = np.mean(eval_rewards)
eval_mean_rewards.append(eval_mean_reward)
print("Mean evaluation reward: {}".format(eval_mean_reward))
if eval_mean_reward > best_eval_mean_reward:
print("Model has improved! Saving the model!")
best_eval_mean_reward = eval_mean_reward
parameter_server.save_eval_weights.remote()
print("Finishing the training.\n\n\n\n\n\n")
[actor.stop.remote() for actor in train_actor_ids]
learner.stop.remote()
class NeuroCuberServer:
def __init__(self, cfg):
self.cfg = cfg
self.tbwriter = TensorBoardWriter(cfg)
self.replay_buffer = ReplayBuffer(cfg)
self.learner_outqueue = queue.Queue()
self.learner = Learner(cfg, replay_buffer=self.replay_buffer, outqueue=self.learner_outqueue)
self.learner_thread = LearnerThread(self.learner)
self.learner_thread.start()
self.learner_post_thread = LearnerPostThread(cfg, self.learner_outqueue, self.tbwriter)
self.learner_post_thread.start()
def get_config(self):
return self.cfg
def get_weights(self):
return self.learner.get_weights()
def process_actor_episode(self, aer):
aer = ActorEpisodeResult(*aer)
assert(aer is not None)
self.tbwriter.log_actor_episode(aer)
self.replay_buffer.add_datapoints(aer.datapoints)
def run():
parser = optparse.OptionParser()
parser.add_option('-p', '--parked', dest='parked', default=False, action='store_true')
parser.add_option('-d', '--display', dest='display', default=True, action='store_true')
parser.add_option('-k', '--numCars', type='int', dest='numCars', default=3)
parser.add_option('-l', '--layout', dest='layout', default='small')
parser.add_option('-s', '--speed', dest='speed', default='slow')
parser.add_option('-f', '--fixedSeed', dest='fixedSeed', default=False, action='store_true')
parser.add_option('-a', '--auto', dest='auto', default=False, action='store_true')
(options, _) = parser.parse_args()
Const.WORLD = options.layout
Const.CARS_PARKED = options.parked
Const.SHOW_CARS = options.display
Const.NUM_AGENTS = options.numCars
Const.INFERENCE = 'none'
Const.AUTO = options.auto
Const.SECONDS_PER_HEARTBEAT = 0.001
signal.signal(signal.SIGINT, signal_handler)
# Fix the random seed
if options.fixedSeed: random.seed('driverlessCar')
learner = Learner()
iterations = 0
numIter = Const.TRAIN_MAX_AGENTS * Const.TRAIN_PER_AGENT_COUNT
for i in range(1, Const.TRAIN_MAX_AGENTS + 1):
for j in range(Const.TRAIN_PER_AGENT_COUNT):
percentDone = int(iterations * 100.0 / numIter)
print str(percentDone) + '% done'
Const.NUM_AGENTS = i
quit = Controller().learn(learner)
if quit:
Display.endGraphics()
return
iterations += 1
transFileName = Const.WORLD + 'TransProb.p'
transFilePath = os.path.join('learned', transFileName)
with open(transFilePath, 'wb') as transFile:
learner.saveTransitionProb(transFile)
print 'saved file: ' + transFilePath
Display.endGraphics()
def learn_wfsa(wfsa, corpus, distfp=None, checkpoint=None):
wfsa_ = copy(wfsa)
wfsa_.round_and_normalize()
if not checkpoint:
checkpoint = lambda x: wfsa_.dump(open('{0}.wfsa'.format(x), 'w'))
if distfp is not None:
if wfsa_.quantizer is not None:
bits = int(round(math.log(wfsa_.quantizer.levels, 2)))
f = [2 ** i for i in xrange(max(0, bits-5), -1, -1)]
t = [1e-5/2**i for i in xrange(0, max(1, bits-4))]
learner = Learner(wfsa_, corpus, checkpoint, pref_prob=0.0,
distfp=distfp, turns_for_each=300, factors=f,
temperatures=t)
else:
# continuous case, not implemented
pass
learner.main()
logging.debug("WFSA learnt")
return wfsa_
def testNearestNeighbor(self): weights = [0] * len(self.board.getMoveList(AI_COLOR)) weights[0] = 1 self.learner = Learner(data_points = [(self.board.getArray().tolist(), weights)]) # self.board.getMoveList(AI_COLOR)[0][1].printMove() # self.learner.getNextMove(self.board).printBoard() self.assertEqual(self.learner.getNextMove(self.board), self.board.getMoveList(AI_COLOR)[0][1], \ 'predicted best move does not match')
def main():
# create AI decision tree
learner1 = Learner(1)
learner2 = Learner(2)
learner2.display_tree()
learner2.cross_validation()
# initialize joysticks
# because this line is only called once, all usuable joysticks must be plugged in at program start
pygame.joystick.init()
joysticks = [pygame.joystick.Joystick(x) for x in range(pygame.joystick.get_count())]
for i in range(len(joysticks)):
joysticks[i].init()
joystick1 = ''
joystick2 = ''
if len(joysticks) > 0:
joystick1 = GameJoystick("xbox", joysticks[0])
else:
joystick1 = GameJoystick("keyboard")
if len(joysticks) > 1:
joystick2 = GameJoystick("xbox", joysticks[1])
else:
joystick2 = GameJoystick("keyobard")
# set screen size
screen = pygame.display.set_mode((screenX, screenY))
pygame.display.set_caption('Falcon Poncho')
fontObj = pygame.font.Font('freesansbold.ttf', 32)
player1 = Player(1, 75, 100, joystick1)
player2 = Player(2, 590, 100, joystick2)
player2.myDirection = direction['left']
world = GameWorld()
recorder = Recorder()
# timer used for falcon punch timing
currentTime = 0
menuTimer = 0
# load music and play indefinitely
menuMusic = pygame.mixer.Sound("lib/Menu_Music.wav")
menuMusic.play()
inMenu = True
canPressStart = False
menuBackground = pygame.image.load("lib/main_menu.png")
startButton = pygame.image.load("lib/press_start.png")
while True:
# Repetition
# check for quit event
for event in pygame.event.get():
if event.type == QUIT:
pygame.quit()
sys.exit()
if inMenu:
screen.blit(pygame.transform.scale(menuBackground, (screenX, screenY)), (0, 0))
if menuTimer < 750:
screen.blit(startButton, (300, 500))
else:
canPressStart = True
if not joystick1 == '':
if joystick1.checkAction('start') and canPressStart:
inMenu = False
menuMusic.stop()
gameMusic.play(-1)
canPressStart = False
if joystick1.checkAction('select') and canPressStart:
player2.isAI = True
player2.learner = learner2
inMenu = False
menuMusic.stop()
gameMusic.play(-1)
canPressStart = False
if joystick1.checkAction('home') and canPressStart:
player1.isAI = True
player1.learner = learner1
player2.isAI = True
player2.learner = learner2
inMenu = False
menuMusic.stop()
gameMusic.play(-1)
canPressStart = False
menuTimer = menuTimer%1500 + fpsClock.get_time()
else:
# check time passed
currentTime += fpsClock.get_time()
GameDriver(player1, player2, world, currentTime, joystick1, joystick2, screen, fontObj, recorder)
# Essential lines of code. Game will run turble if these are not here
pygame.display.update()
fpsClock.tick(60) # 60 fps??? Can't really tell
def handle_command(command, details, channel, respond = True):
"""
Receives commands directed at the bot and determines if they
are valid commands. If so, then acts on the commands. If not,
returns back what it needs for clarification.
"""
response = False
if command == "learn":
learner = Learner()
response = learner.learn(details[0], " ".join(details[1:]))
elif command == "unlearn":
learner = Learner()
content = None
if len(details) > 1:
content = " ".join(details[1:])
response = learner.unlearn(details[0], content)
elif command == "commands":
learner = Learner()
response = learner.list_commands()
elif command == "list":
learner = Learner()
response = learner.list(details[0])
elif command == "cowsay":
out = subprocess.check_output(['cowsay', " ".join(details)])
response = "```" + out + "```"
elif command == "meme":
memer = Memer()
if not details or len(details) == 0:
response = memer.list_templates()
else:
template = details.pop(0).strip()
parts = [x.strip() for x in " ".join(details).split(",")]
top = parts[0] if len(parts) > 0 else None
bottom = parts[1] if len(parts) > 1 else None
response = memer.get_meme(template, top, bottom)
elif command == "hostname":
response = "slurms coming to you live from: `%s (%s)`" % (subprocess.check_output("hostname -A", shell=True).strip(), subprocess.check_output("hostname -i", shell=True).strip())
elif command == "write":
writer = Writer()
response = writer.get_writing(" ".join(details))
elif command == "imglearn":
learner = Learner()
imgur = Imgur()
image_url = imgur.save_from_url(" ".join(details[1:]))
response = learner.learn(details[0], image_url)
elif command == "++" or command == "endorse":
plusser = Plusser()
reason = ""
if len(details) > 1:
reason = " ".join(details[1:])
response = plusser.plus(details[0], reason)
elif command == "plusses":
plusser = Plusser()
response = plusser.get(details[0])
elif command == "leaders" or command == "leader_board":
plusser = Plusser()
response = plusser.leader_board()
elif command == "monthly_leaders" or command == "monthly_leader_board":
plusser = Plusser()
months_ago = 0
if details and len(details) > 0:
months_ago = details[0]
response = plusser.monthly_leader_board(months_ago)
elif command == "youtube":
query = " ".join(details)
videos = youtube.youtube_search(query)
if len(videos) > 0:
response = videos[-1]
else:
response = "sorry, couldnt find any videos for %s" % query
elif command == "echo":
response = " ".join(details)
elif command == "pipe":
pipe(command, details, channel)
elif command == "doom":
doom = Doom()
response = doom.doom(details)
else:
"""
see if a randomly entered command is something that was previously learned
"""
learner = Learner()
response = learner.get(command)
if response and respond:
slack_client.api_call("chat.postMessage", channel=channel,
text=response, as_user=True)
elif not respond:
return response
def __init__(self,initialTraining):
"""Constructor"""
self.initialTraining = initialTraining
self.States = []
self.Actions = []
self.learner = Learner()
class Dagger():
""" In fact the Python twin of the
corresponding Java ForwardAgent.
"""
def __init__(self,initialTraining):
"""Constructor"""
self.initialTraining = initialTraining
self.States = []
self.Actions = []
self.learner = Learner()
def getName(self):
return 'Dagger'
def loadModel(self):
self.learner.Load()
def askForHelp(self,img):
return 1
def getAction(self,img):
""" Possible analysis of current observation and sending an action back
"""
action = self.learner.getAction(img)
self.actionTaken = action
return action
def integrateObservation(self, img,action):
"""This method stores the observation inside the agent"""
if (self.initialTraining or (self.actionTaken[0] != action[0] and self.action[0] != 2)):
img = cv2.pyrDown((cv2.pyrDown(img)))
winSize = (32,32)
blockSize = (16,16)
blockStride = (8,8)
cellSize = (8,8)
nbins = 9
derivAperture = 1
winSigma = 4.
histogramNormType = 0
L2HysThreshold = 2.0000000000000001e-01
gammaCorrection = 0
nlevels = 64
hog = cv2.HOGDescriptor(winSize,blockSize,blockStride,cellSize,nbins,derivAperture,winSigma,
histogramNormType,L2HysThreshold,gammaCorrection,nlevels)
state = hog.compute(img)
self.States.append(state)
self.Actions.append(action)
#self.printLevelScene()
def updateModel(self):
States = np.array(self.States)
Actions = np.array(self.Actions)
self.learner.trainModel(States,Actions,fineTune = True)
def getDataAdded(self):
return self.dataAdded
def newModel(self):
states = np.array(self.States)
actions = np.array(self.Actions)
self.learner.trainModel(states,actions)
def getNumData(self):
return self.learner.getNumData()
def reset(self):
self.States = []
self.Actions = []
def __init__(self, module_properties, dao):
Learner.__init__(self, module_properties, dao, SGDClassifier(loss=module_properties['loss'], penalty=module_properties['penalty'],
learning_rate=module_properties['step_policy'], eta0=module_properties['eta0'], average=module_properties['average'],
shuffle=False))
def __init__(self): self.learn = learn self.load = load self.learner = Learner() if self.load and os.path.isfile(learnerFile): self.learner.loadData(learnerFile) self.learner.newGame()
logging.info('grammar node type num.: %d', len(train_data.grammar.node_type_to_id))
logging.info('source vocab size: %d', train_data.annot_vocab.size)
logging.info('target vocab size: %d', train_data.terminal_vocab.size)
if args.operation in ['train', 'decode', 'interactive']:
model = Model()
model.build()
if args.model:
model.load(args.model)
if args.operation == 'train':
# train_data = train_data.get_dataset_by_ids(range(2000), 'train_sample')
# dev_data = dev_data.get_dataset_by_ids(range(10), 'dev_sample')
learner = Learner(model, train_data, dev_data)
learner.train()
if args.operation == 'decode':
# ==========================
# investigate short examples
# ==========================
# short_examples = [e for e in test_data.examples if e.parse_tree.size <= 2]
# for e in short_examples:
# print e.parse_tree
# print 'short examples num: ', len(short_examples)
# dataset = test_data # test_data.get_dataset_by_ids([1,2,3,4,5,6,7,8,9,10], name='sample')
# cProfile.run('decode_dataset(model, dataset)', sort=2)
def setUp(self): self.board = Board() self.learner = Learner()
def __init__(self, kb, n=None, min_sup=1, sim=1, depth=4, target=None,
use_negations=False, optimal_subclass=True):
Learner.__init__(self, kb, n=n, min_sup=min_sup, sim=sim, depth=depth,
target=target, use_negations=use_negations)
def __init__(self, module_properties, dao):
Learner.__init__(self, module_properties, dao, PassiveAggressiveClassifier(C=module_properties['C'],
loss=module_properties['loss'], shuffle=False))
class KMeansAgent(Agent):
"""
K-Means agent is built on the same idea that we can extract features
from K-means in the image recognition.
Unfortunately, this agent cannot get better as the euclidean distance
between states is meaningless !
"""
def __init__(self):
self.learn = learn
self.load = load
self.learner = Learner()
if self.load and os.path.isfile(learnerFile): self.learner.loadData(learnerFile)
self.learner.newGame()
def getAction(self, gameState):
"""
The selected action is simply the best action computed by the
(not that) smart evaluationFunction.
"""
# Before trying to find an action, we save the gameState
#print flattenGameState(gameState)
self.learner.addState(flattenGameState(gameState))
# Collect legal moves and successor states
legalMoves = gameState.getLegalActions()
legalMoves.remove('Stop')
# Choose one of the best actions
scores = [self.notThatSmartEvaluationFunction(gameState, action) for action in legalMoves]
#print scores
bestScore = max(scores)
bestIndices = [index for index in range(len(scores)) if scores[index] == bestScore]
chosenIndex = random.choice(bestIndices) # Pick randomly among the best
return legalMoves[chosenIndex]
def final(self,finalState):
self.learner.labelizeData(finalState)
if self.learner.gamesInMemory % processingFrequency == 0:
self.learner.processLearning()
if saveFile:
print 'Saving learner file...'
self.learner.saveData(learnerFile)
#self.learner.saveData(learnerFile)
self.learner.newGame()
#flattenGameState(finalState)
def notThatSmartEvaluationFunction(self, currGameState, pacManAction):
nextGameState = currGameState.generatePacmanSuccessor(pacManAction)
if self.learner.readyToPredict == False:
return nextGameState.getScore()
else:
nextGameState = currGameState.generatePacmanSuccessor(pacManAction)
stateFeatures = self.learner.extractFeatures(flattenGameState(nextGameState))
predictedScore = self.learner.predictScore(stateFeatures)
#print predictedScore
return float(predictedScore)
def trainWithLoser(winner, game_count): loser = Learner() # game_board = Board() for game in tqdm(range(game_count)): game_board = Board() winner_moves = [] loser_moves = [] game_history = [] turn_count = 0 tie_flag = False while game_board.checkGameStatus(AI_COLOR) == CONTINUE: if turn_count > 100: tie_flag = True break loser_move = loser.getNextMove(game_board.getInverse()) loser_moves.append(loser_move.getInverse()) assert(loser_move is not None and game_board is not None) game_history.append((game_board, loser_move.getInverse())) # loser_move.printMove() # loser_move.getInverse().printMove() # game_board.printBoard() # temp = game_board # game_board.printBoard() game_board = Board(game_board.applyMove(loser_move.getInverse())) # game_board.printBoard() # assert(temp != game_board) # loser_move.getInverse().printMove() # print turn_count # game_board.printBoard() if game_board.checkGameStatus(AI_COLOR) == LOSE: break # assert (temp != game_board) winner_move = winner.getNextMove(game_board) assert(winner_move is not None and game_board is not None) game_history.append((game_board, winner_move)) # winner_move.printMove() winner_moves.append(winner_move) # temp = game_board game_board = Board(game_board.applyMove(winner_move)) # game_history.append(game_board) # game_board.printBoard() turn_count += 1 # print game if not tie_flag and game_board.checkGameStatus(AI_COLOR) != TIE: winner.updateWeights(game_history, status = game_board.checkGameStatus(AI_COLOR)) # winner.updateWeights(game_board, loser_moves, winner_moves, game_history = game_history) else: winner.updateWeights(game_history, status = TIE) # game_board.printBoard() # winner.updateWeights(game_board, loser_moves, winner_moves, status = TIE, game_history = game_history) loser_moves = None winner_moves = None return winner
def fit(self): self._half_feature_vectors = self.feature_vectors.copy() Learner.fit(self) self._half_feature_vectors[:,self.half_index:] = 0 self._half_feature_vectors = self.preprocess(self._half_feature_vectors)
