def get_data(path, train_size=456):
dt = Container()
DT = []
with open(path, 'rb') as f:
for line in f:
line = [float(val) for val in line.strip().split()]
DT.append(line)
DT = np.asarray(DT).astype(theano.config.floatX)
# shuffle
shuffle_idx = np.random.permutation(DT.shape[0])
DT = DT[shuffle_idx]
DT_x = DT[:, :-1]
DT_y = DT[:, -1]
# data standardization
dt.mu = np.mean(DT_x, axis=0)
dt.sig = np.std(DT_x, axis=0)
DT_x = (DT_x - dt.mu) / dt.sig
dt.x_train = DT_x[:train_size]
dt.y_train = DT_y[:train_size]
dt.x_val = DT_x[train_size:]
dt.y_val = DT_y[train_size:]
return dt
def normalizeAmountField(self, amount_text, row):
is_usa_donation = not (row['AccountAlias'] == 'Adrian Weisensee')
if row['TransferMech1'] == 'Donation Received':
amount = Container.getPaypalAdjusted(row['Amount'], is_usa_donation)
else:
amount = row['Amount']
return amount
def parse_msg(self, msg):
msg = msg[1:-1] # removes delimeters
(id_robot, seq_number, information) = msg.split('/')
(type_intersection, distance) = information.split(';')
parsed_msg = {}
parsed_msg['id_robot'] = int(id_robot)
parsed_msg['seq_number'] = int(seq_number)
parsed_msg['type_intersection'] = int(type_intersection)
parsed_msg['distance'] = self.discretize(float(distance))
return Container(parsed_msg)
def send_information_to_master(self, msg):
directive = {
"type_directive": "information",
"id_robot": msg.id_robot,
"information": {
"type_intersection": msg.type_intersection,
"distance": msg.distance
}
}
self.q.messenger2master.put(Container(directive))
def build_communication(self):
self.id_master = 0
self.n_robots = self.params.n_robots
self.robots = {}
for i in range(self.n_robots):
id_robot = i + 1
comm = {}
comm["seq_number"] = 0
self.robots[id_robot] = comm
self.robots = Container(self.robots)
def build_communication(self):
self.id_master = 0
self.n_robots = self.params.n_robots
self.robots = {}
for robot_id, robot_comm in self.params.robots.items():
temp = {}
temp["serial"] = serial.Serial(robot_comm["port"],
robot_comm["baud_rate"],
timeout=0.35)
temp["seq_number"] = 0
self.robots[robot_id] = temp
self.robots = Container(self.robots)
def __init__(self, model, env, **kwargs):
Agent.__init__(self, **kwargs)
self.update_step = 0
self.eps = self.EPS_START
self.global_step = 0
self.model = model
self.target_model = copy.deepcopy(model)
self.in_size = model.in_size
self.out_size = model.out_size
self.memory = ReplayMemory(self.REPLAY_CAPACITY)
self.opt = torch.optim.Adam(self.model.parameters(), lr=self.LR)
self.env = env
self.container = Container(self.model.SAVE_MODEL_NAME)
def make_queues():
q_master = {
"master2gui": Queue(),
"gui2master": Queue(),
"robots2messenger": Queue(),
"messenger2robots": Queue()
}
q_gui = {
"master2gui": q_master["master2gui"],
"gui2master": q_master["gui2master"]
}
q_robots = {
"robots2messenger": q_master["robots2messenger"],
"messenger2robots": q_master["messenger2robots"]
}
q = {"master": q_master, "robots": q_robots, "gui": q_gui}
return Container(q)
def get_data(path, train_size=50000):
dt = Container()
DT = []
with open(path, 'rb') as f:
if (path.endswith('.csv')):
for line in f:
line = [float(val) for val in line.strip().split()]
DT.append(line)
# shuffle
shuffle_idx = np.random.permutation(DT.shape[0])
DT = DT[shuffle_idx]
DT_x = DT[:, :-1]
DT_y = DT[:, -1]
dt.x_train = DT_x[:train_size]
dt.y_train = DT_y[:train_size]
dt.x_val = DT_x[train_size:]
dt.y_val = DT_y[train_size:]
return dt
(3, "baseball bat", 1),
(5, "whale", 10),
(6, "city block", 100),
(7, "Eiffel tower", 300),
(8, "English channel (narrowest)", 35000),
(9, "AU (Earth to Sun)", 1.5e11),
(10, "Light Year", 9.5e15),
(21, "thickness of sunglasses", 0.001),
(22, "Staphilococcus bacterium", 1e-6),
(23, "Poliovirus", 3e-8),
(24, "Hydrogen atom diameter", 1e-10),
(25, "Hydrogen nucleus", 2.4e-15),
]
sizes = {s[0]: Container(name=s[1], size=s[2]) for s in sizes}
class Rescaler(object):
tpl = "%-28s %s"
msg = "If %s was the size of %s" + nl*2
def rescale(self, i1, i2):
i1, i2 = sizes[i1], sizes[i2]
scaled = copy(sizes)
ratio = i1.size / i2.size
print(self.msg % (i1.name, i2.name))
for item in scaled.values():
item.size /= ratio
print(self.tpl % (item.name, self.format(item.size)))
def send_user_request_to_master(self, request):
directive = {"type_directive": "request_" + request}
self.q.gui2master.put(Container(directive))
def determineDerivedFields(self, field, useful, derived):
is_amount, amount = self.determineAmount(field, useful, derived)
if is_amount:
return amount
elif (field == 'Date'):
return self.normalizeDateField(useful[field])
elif (field == 'Credit'):
amount = Container.getNegativeAmounts(useful['Amount'])
no_commas = re.sub(',','', amount)
if float(no_commas) >= 0.0:
return amount
else:
return ''
elif (field == 'Debit'):
amount = Container.getNegativeAmounts(useful['Amount'])
no_commas = re.sub(',','', amount)
if float(no_commas) < 0.0:
return amount
else:
return ''
elif (field == 'Year'):
return Container.convertDateToYear(useful['Date'])
elif (field == 'YearMonth'):
return Container.convertDateToYearMonth(useful['Date'])
elif (field == 'Mechanism'):
mech = self.getMethodOfPayment(
useful['TransferMech1'], useful['TransferMech2'] )
if useful['TransferMech1'] == "Checking":
show_check_number = re.sub(
'Check','Paper Check ',useful['TransferMech2'])
return show_check_number
if useful['TransferMech2'] == "Bill Payment":
show_check_number = re.sub(
'THE LION OF JUDA00','ML BillPay Check ',useful['AccountAlias'])
return show_check_number
return mech
elif (field == 'SimplifiedAlias'):
self.simplified_alias = re.sub('[0123456789]+', '', useful['AccountAlias'])
self.simplified_alias = Container.String(self.simplified_alias).removeBlanks()
return self.simplified_alias
elif (field == 'Account'):
self.account_name = self.getAccountTitle(
useful['AccountAlias'],
useful['TransferMech1'],
useful['TransferMech2'],
)
return self.account_name
elif (field == 'Institution'):
return self.institutionName()
elif (field == 'CenterAccount'):
return self.centerAccount()
elif (field == 'OrbitAccount'):
return derived['Account']
elif (field == 'FromAccount'):
if Container.getFloatNoCommas(self.normalizeAmountField(useful['Amount'], useful)) < 0.0:
return self.centerAccount()
else:
return derived['Account']
elif (field == 'ToAccount'):
if Container.getFloatNoCommas(self.normalizeAmountField(useful['Amount'], useful)) < 0.0:
return derived['Account']
else:
return self.centerAccount()
elif (field == 'Subcategory'):
return self.getSubcategory(self.account_name)
elif (field == 'Category'):
return self.getCategory(self.account_name)
else:
return useful[field]
def __init__(self):
assert self._instance==None
self._instance=self
Container.__init__(self)
self.__initialised=False
def train(self, model, data, params={}):
'''Main training loop.
Arguments:
model: object / Container (see utils.py)
The model. It should have at least the following attributes:
model.inputs: theano.tensor.matrix
Representing input minibatch.
model.outputs: theano.tensor.row
Representing model outputs.
model.weights: list of theano.shared variables
List of model parameters.
data: object / Container
The data. It should have at least the following attributes:
data.x_train: np.array
data.y_train: np.array
data.x_val: np.array
data.y_val: np.array
params: dict
Additional parameters for training.
'''
self.inputs = model.inputs
self.model_outputs = model.outputs
self.weights = model.weights
self.initialize_params(params, data)
self._create_auxiliary_variables()
# get update equations
self.updates, self.sumloglik = self._get_updates()
n = self.params['batch_size']
N = self.params['train_size']
lr = self.params['lr']
# create training and prediction functions
fn = Container()
fn.train = self._get_training_function()
fn.predict = self._get_prediction_function()
avg_pp = np.zeros(data.y_val.shape)
sum_pp = np.zeros(data.y_val.shape)
sumsqr_pp = np.zeros(data.y_val.shape)
n_samples = 0
do_sampling = False
for i in range(self.params['n_iter']):
# prepare next minibatch
mini_idx = np.floor(np.random.rand(n) * N).astype('int32')
mini_X = data.x_train[mini_idx]
mini_Y = data.y_train[mini_idx]
# parameter update
train_pp, sumloglik = fn.train(mini_X, mini_Y, lr)
if i % self.params['lr_decay'] == 0:
lr /= 2
if i == self.params['burn_in']:
# burnin period over, begin sampling
do_sampling = True
if i % self.params['thinning'] == 0:
val_pp = fn.predict(data.x_val)
if do_sampling:
n_samples += 1
# prediction based on current parameter (sample)
avg_pp = ((1 - (1. / n_samples)) * avg_pp) + (
(1. / n_samples) * val_pp)
# trn_pp = fn.predict(data.x_train) # train predictions
ppp = avg_pp
# online sample variance
sum_pp += val_pp
sumsqr_pp += val_pp * val_pp
var_pp = (sumsqr_pp -
(sum_pp * sum_pp) / n_samples) / (n_samples - 1)
else:
ppp = val_pp
trn_pp = fn.predict(data.x_train) # train predictions
var_pp = np.var(trn_pp - data.y_train)
meanloglik = logpdf_normal(ppp, data.y_val, 1 / var_pp).mean()
rmse = compute_rmse(ppp, data.y_val)
# trmse = compute_rmse(trn_pp, data.y_train)
print ('%d/%d, %.2f, %.2f (%.2f) \n' % \
(i, n_samples, sumloglik, meanloglik, rmse), end = "")
print ('%d/%d, %.2f, %.2f (%.2f)' % \
(i, n_samples, sumloglik, meanloglik, rmse))
def normalizeDateField(self, date_string):
return Container.convertDateWithSlashes(date_string)
def __init__(self, n_battlefields, e=0.05):
self.policy = Container(n_battlefields) # OK
self.returns = Container(n_battlefields + 1, [])
self.action_value = Container(n_battlefields + 1, 0)
self.e_greedy = e
self.discount = 0.1
std_time = []
for lambda_ in np.linspace(0, 2, 21):
times = []
n_failures = 0
for i in range(10):
q = make_queues()
env = Environment()
gt = set(env.ground_truth)
robot1 = Robot(1, q.robots, env)
robot2 = Robot(2, q.robots, env)
master = NaiveMaster(params, q.master, lambda_=lambda_)
robot1.start()
robot2.start()
master.start()
directive = {"type_directive": "request_run"}
q.gui.gui2master.put(Container(directive))
edges = {}
start_time = time.time()
while True:
if not q.gui.master2gui.empty():
directive = q.gui.master2gui.get_nowait()
edges = set(directive.summary.edges)
common = gt.intersection(edges)
overlap = float(len(common)) / float(len(gt))
time.sleep(0.05)
current_time = time.time()
if current_time - start_time > 60 or master.finished:
break
if master.finished:
times.append((current_time - start_time) / len(gt))
else:
from modules import TwitterModule, ClockModule, WeatherModule, StockModule from utils import Container container = Container() container.add_module(ClockModule, "left") container.add_module(TwitterModule, "bottom") container.add_module(WeatherModule, "right") container.add_module(StockModule, "rbot", stocks=["AAPL", "TSLA", "GOOGL"]) container.start()
def main(filename='2012-05-31 IRO2012-Pre2/patches.json'):
patches = Container(filename)
plot_error(patches)
#plt.tight_layout()
plt.show()
back of the flash card.
Note: flashcard file entries need to be separated by double dashes (see `sep` setting)
"""
import sys
from os.path import exists
from textwrap import wrap
from time import sleep
from copy import copy
from utils import TextInput, Container, shuffled, getitem, nl, space
width = 78
screensep = 30
border = Container(tl='╭', tr='╮', bl='╰', br='╯', horiz='─', vertical='│')
cards_fn = "cards.txt"
question = " Did you get it right (Y/n)? "
status = "\n %d right out of %d (%d%%)\n"
sep = "--"
textinput = Container(question=TextInput(accept_blank=True, prompt=question),
pause=TextInput(accept_blank=True))
class Card(object):
def __init__(self, card):
front, back = card.split(sep, 1)
self.front = front.strip()
self.back = back.strip()
def normalizeAmountField(self, amount_text, row):
return Container.getNegativeAmounts(amount_text)
from random import choice as randchoice
from collections import defaultdict
from copy import copy
from utils import Loop, Container, TextInput, first, sjoin, nl, space
from board import StackableBoard, Loc, BaseTile
commands = dict(a="left",
s="back",
w="forward",
d="right",
p="pickup",
i="inventory",
m="map",
l="look")
roomchance = Container(door=0.8, shaky_floor=0.1)
itemchance = Container(Gem=0.1, Key=0.05, Gold=0.25, Anvil=0.01)
itemchance = Container(Gem=0.1, Key=0.1, Gold=0.1, Anvil=0.2)
crystalchance = 0.1
size = 15
lhoriz = '─'
lvertical = '│'
doorchar = '⌺'
roomchar = '▢'
player_char = '☺'
absdirs = range(4)
class Item(BaseTile):
gem = key = gold = anvil = crystal = False
