def _task_create_kwargs(arguments):
result = AttrDict(
name=arguments['<name>'],
state='pending',
command=arguments['--command'],
condition=arguments['--condition'],
last=str(datetime.now()),
)
if arguments['--once']:
result.last += '<once>'
if arguments['--inherit']:
parent = db.read(table='tasks', name=arguments['--inherit'])
result.update(dict(
parent=parent.name,
schedule='<inherit>',
command=result.command if result.command else '<inherit>',
email='<inherit>',
condition=result.condition if result.condition else '<inherit>',
))
result.update(_task_sched_kwargs(arguments))
return result
def run_lstm_variant(variant='SlowLSTM', cuda=False, size=128, jit=False):
assert variant in lstms
p = AttrDict({'cuda': cuda, 'lstm_kind': variant, 'size': size})
name = '{}_size{}{}{}'.format(variant, size, tag(cuda=cuda), tag(jit=jit))
def C(x):
if p.cuda:
x = x.cuda()
return x
lstm = getattr(lstm_variants, p.lstm_kind)
x = V(C(th.rand(1, BATCH, p.size)))
hiddens = (V(C(th.rand(1, BATCH, p.size))), V(C(th.rand(1, BATCH,
p.size))))
th.manual_seed(1234)
cus = C(lstm(p.size, p.size, dropout=DROPOUT, jit=jit))
if hasattr(cus, 'mask'):
cus.mask = C(cus.mask)
iter_timer = Bench(name=name, cuda=cuda, warmup_iters=3)
# Super slow on CPU
iters = 20 if cuda else 6
for _ in range(iters):
gc.collect()
with iter_timer:
out, h = x, hiddens
for i in range(SEQ_LEN):
out, h = cus(out, h)
return iter_timer
def read() -> AttrDict:
with open(SETTINGS_DEFAULT) as f:
settings = AttrDict(yaml.load(f))
try:
with open(SETTINGS_USER) as f:
setting_dict = yaml.load(f)
except FileNotFoundError:
open(SETTINGS_USER, 'w').close()
else:
if setting_dict is not None:
settings.update(setting_dict)
return settings
def _print_subtask(task, long, indent):
task = AttrDict(task)
tabs = ' ' * indent
print(tabs, end='')
if long:
print(task.name, ', '.join('{}: {}'.format(k, v) for k, v in task.items()
if v and k != 'name' and k[0] != '~'),
end='')
else:
print('{}: {}'.format(task.name, task.state), end='')
if '~summary' in task:
print('', _subtask_summary_repr(task['~summary']))
else:
print()
def time_a_star(h, w, p, maps, runs):
data = AttrDict({'h': l1_dist})
generate_map = lambda: generate_random_map(h, w, p=p, start_and_goal=False)
algo = lambda m, s, g, d: a_star(m, s, g, d.h)
avg_time, times, fc = experiment(algo,
generate_map=generate_map,
data=data,
maps=maps,
runs=runs)
print('%e' % avg_time, fc)
def time_block_a_star(b_sz, h, w, p, maps, runs):
lddb, pathsdb = make_lddb(b_sz, from_file=True, save_to_file=False)
data = AttrDict({
'block_size': b_sz,
'lddb': lddb,
'pathsdb': pathsdb,
'h': l1_dist
})
generate_map = lambda: generate_random_map(h, w, p=p, start_and_goal=False)
preprocessing = lambda m, d: BlockMap(m, d.block_size)
algo = lambda m, s, g, d: block_a_star(d.lddb, d.pathsdb, m, s, g, d.h)
avg_time, times, fc = experiment(algo,
generate_map=generate_map,
data=data,
preprocessing=preprocessing,
maps=maps,
runs=runs)
print('%e' % avg_time, fc)
def experiment(algo,
generate_map,
data=AttrDict(),
preprocessing=lambda m, d: m,
maps=500,
runs=100,
ignore_failures=False):
times = []
failure_count = 0
for i in range(maps):
if i % 10 == 0:
print(i)
_m = generate_map()
m = _m.copy()
h, w = _m.shape
starts = [(np.random.randint(h), np.random.randint(w))
for _ in range(runs)]
goals = [(np.random.randint(h), np.random.randint(w))
for _ in range(runs)]
for j, (start, goal) in enumerate(zip(starts, goals)):
m = m | _m
m[start[0], start[1]] = 0
m[goal[0], goal[1]] = 0
Map = preprocessing(m, data)
t1 = perf_counter()
goal_found, path = algo(Map, start, goal, data)
times.append(perf_counter() - t1)
if not goal_found:
failure_count += 1
if ignore_failures:
continue
return sum(times) / len(times), times, failure_count
def aggregateInstances(instances):
"""Aggregates instances in to a meta instance."""
result = AttrDict(
count = 0,
firstOccurrence = None,
lastOccurrence = None,
lastMessage = None,
backtrace = None,
environments = set(),
servers = set()
)
for instance in instances:
aggregate(result,
int(instance['count']),
parseDate(instance['firstOccurrence']),
parseDate(instance['lastOccurrence']),
instance['lastMessage'],
instance['backtrace'],
instance['environments'],
instance['servers'])
return result
self.data_batches = [
Variable(cast(torch.zeros(p.batch_size, 28 * 28, 1)))
for _ in range(p.num_batches)
]
self.target_batches = [
Variable(cast(torch.zeros(p.batch_size)).long())
for _ in range(p.num_batches)
]
if p.cuda:
self.model.cuda()
self.criterion.cuda()
def time(self, p):
total_loss = 0
for data, targets in zip(self.data_batches, self.target_batches):
logits = self.model(data)
loss = self.criterion(input=logits, target=targets)
loss.backward()
total_loss += loss.data # CUDA sync point
if p.cuda:
torch.cuda.synchronize()
if __name__ == '__main__':
d = WLM.default_params.copy()
d['cuda'] = False
p = AttrDict(d)
m = WLM()
m.prepare(p)
m.time(p)
def run_memnn(warmup=2, benchmark=18, jit=False, cuda=False):
nbatches = warmup + benchmark
default_params = dict(lr=0.01,
embedding_size=128,
hops=3,
mem_size=100,
time_features=False,
position_encoding=True,
output='rank',
dropout=0.1,
optimizer='adam',
num_features=500,
num_batches=nbatches,
cuda=cuda)
params = AttrDict(default_params)
"""Set up model."""
# The CPU version is slow...
params['batch_size'] = 4 if params.cuda else 4
if params.cuda:
device = torch.device('cuda:0')
else:
device = torch.device('cpu')
model = memnn.MemNN(params, params.num_features)
criterion = nn.CrossEntropyLoss()
data_batches = [
[ # memories, queries, memory_lengths, query_lengths
torch.zeros(params.batch_size * params.mem_size,
dtype=torch.long,
device=device),
torch.zeros(params.batch_size * 28,
dtype=torch.long,
device=device),
torch.ones(params.batch_size,
params.mem_size,
dtype=torch.long,
device=device),
torch.full((params.batch_size, ),
28,
dtype=torch.long,
device=device),
] for _ in range(params.num_batches)
]
cand_batches = [
torch.zeros(params.batch_size * 14,
params.embedding_size,
device=device) for _ in range(params.num_batches)
]
target_batches = [
torch.ones(params.batch_size, dtype=torch.long, device=device)
for _ in range(params.num_batches)
]
# model.to(device) # embeddings are performed on CPU
# the memnn model takes care of things when it is passed the cuda flag
criterion.to(device)
"""Time model."""
cuda_tag = '_cuda' if cuda else ''
jit_tag = '_jit' if jit else ''
name = 'memnn{}{}'.format(cuda_tag, jit_tag)
bench = Bench(name=name, cuda=cuda, warmup_iters=warmup)
trace_once = jit
total_loss = 0
for data, cands, targets in zip(data_batches, cand_batches,
target_batches):
gc.collect()
if trace_once:
model = torch.jit.trace(*data)(model)
trace_once = False
with bench:
output_embeddings = model(*data)
scores = one_to_many(output_embeddings, cands)
loss = criterion(scores, targets)
loss.backward()
total_loss += float(loss.item())
return bench
def block_a_star(lddb, pathsdb, Map, start, goal, h):
"""Block A* search. Implementation of Algorithm 2 from the paper:
Yap, P., Burch, N., Holte, R. C., & Schaeffer, J. (2011, August).
Block A*: Database-driven search with applications in any-angle path-planning.
In Twenty-Fifth AAAI Conference on Artificial Intelligence.
Args:
lddb: local distance database
pathsdb: local paths database
Map: a BlockMap representing the map to be searched
start: global address of the start node
goal: global address of the goal node
h: heuristic function
Returns:
A tuple of: a boolean value indicating if the goal was found
and a list of nodes on the path between start and goal.
(goal_found, path)
"""
# a dict that will hold the state during the algorithm's run
state = AttrDict({
'Map': Map,
'start': start,
'goal': goal,
'lddb': lddb,
'pathsdb': pathsdb,
'h': lambda block, node: h(to_global_node(block, node), goal),
'g': defaultdict(dict),
'g_changed': defaultdict(dict),
'heapvalue': {},
'heap': PriorityQueue(),
'parent': {}
})
# *_block_node = local address within corresponding block
start_block, start_block_node = Map.get_node_block(start)
goal_block , goal_block_node = Map.get_node_block(goal)
init(state, start_block, start_block_node)
init(state, goal_block, goal_block_node)
state.g[start_block][start_block_node] = 0
state.g_changed[start_block][start_block_node] = True
state.parent[(start_block, start_block_node)] = (None, None)
state.heap.push(start_block, 0)
state.heapvalue[start_block] = 0
length = np.inf
while not state.heap.empty() and state.heapvalue[state.heap.top()[0]] < length:
curr_block = state.heap.pop()
ingress_nodes = get_ingress_nodes(state, curr_block)
if len(ingress_nodes) == 0:
continue
if curr_block == goal_block:
block_lddb = state.lddb[curr_block.idx]
dists_to_goal = [
state.g[curr_block][y] + block_lddb.get((y, goal_block_node), np.inf) for y in ingress_nodes
]
new_length = min(length, np.min(dists_to_goal))
if new_length < length:
length = new_length
nearest_ingress_node = ingress_nodes[np.argmin(dists_to_goal)]
# set parent of goal node, but avoid pointing to self
if goal_block_node != nearest_ingress_node:
state.parent[(goal_block, goal_block_node)] = (curr_block, nearest_ingress_node)
expand_block(state, curr_block, ingress_nodes)
if length < np.inf:
return True, recover_path(state, goal_block, goal_block_node)
else:
return False, []