def repair(G, pheromone_add, pheromone_decay, explore_prob, explore2, strategy='rank',\
num_ants=100, max_steps=1000, num_iters=1, print_graph=False, video=False, \
nframes=-1, video2=False, cost_plot=False, backtrack=False, \
decay_type='exp', node_queue_lim=1, edge_queue_lim=1, one_way=False):
""" """
graph_name = G.graph['name']
nests = G.graph['nests']
out_items = ['repair', strategy, graph_name, decay_type]
if backtrack:
out_items.append('backtrack')
if one_way:
out_items.append('one_way')
out_str = '_'.join(out_items)
def next_destination(origin):
idx = nests.index(origin)
idx += 1
idx %= len(nests)
return nests[idx]
num_edges = G.size()
nframes = min(nframes, max_steps)
pher_str = "%d, %f, %f, " % (num_ants, explore_prob, pheromone_decay)
data_file = open('ant_%s%d.csv' % (out_str, max_steps), 'a')
pruning_file = open('ant_%s_pruning.csv' % out_str, 'a')
critical_nodes_file = open('critical_nodes_%s.csv' % graph_name, 'w')
if video:
fig = PP.figure()
init_path = G.graph['init_path']
track_pruning = max_steps <= MAX_PRUNING_STEPS
for iter in xrange(num_iters):
nonzero_edges = set()
for u, v in G.edges():
G[u][v]['weight'] = MIN_PHEROMONE
G[u][v]['units'] = []
for u, v in init_path:
G[u][v]['weight'] += pheromone_add * INIT_WEIGHT_FACTOR
if decay_type == 'linear':
G[u][v]['units']+= [pheromone_add] * INIT_WEIGHT_FACTOR
nonzero_edges.add(Ninv[(u, v)])
clear_queues(G)
if iter == 0 and print_graph:
pass #color_graph(G, 'g', pheromone_thickness, "graph_before")
print str(iter) + ": " + pher_str
explore = defaultdict(bool)
prevs = {}
prevs2 = {}
prevs3 = {}
currs = {}
paths = {}
walks = {}
destinations = {}
origins = {}
edge_weights = defaultdict(list)
deadend = {}
search_mode = defaultdict(lambda: False)
costs = []
path_counts = defaultdict(int)
chosen_walk_counts = defaultdict(int)
max_entropy = None
max_walk_entropy = None
connect_time = -1
queued_nodes = set()
queued_edges = set()
max_cycles = 0
curr_max_cycles = 0
max_path_len = None
max_chosen_path_len = None
chosen_cycle_counts = []
max_chosen_cycle_count = None
for ant in xrange(num_ants):
origin = nests[ant % len(nests)]
origins[ant] = origin
destinations[ant] = next_destination(origin)
prev_curr = None
if one_way:
prev, curr = init_path[choice((len(init_path) // 2))]
else:
prev, curr = init_path[choice(len(init_path))]
if random() <= 0.5:
prev, curr = curr, prev
paths[ant] = [prev, curr]
if destinations[ant] in paths[ant]:
origins[ant], destinations[ant] = destinations[ant], origins[ant]
walks[ant] = [prev, curr]
prevs[ant] = prev
prevs2[ant] = None
prevs3[ant] = None
currs[ant] = curr
deadend[ant] = False
queue_ant(G, curr, ant)
queued_nodes.add(curr)
steps = 1
rounds = 1
max_weight = MIN_PHEROMONE
unique_weights = set()
max_cost = 0
costs = []
curr_path_entropy = None
curr_walk_entropy = None
for u, v in sorted(G.edges()):
index = Ninv[(u, v)]
wt = G[u][v]['weight']
#unique_weights.add(wt)
max_weight = max(max_weight, wt)
if video:
if wt <= MIN_DETECTABLE_PHEROMONE:
edge_weights[index].append(None)
else:
edge_weights[index].append(wt)
critical_edges_file = None
if 'critical_node' in G.graph:
critical_edges_file = open('critical_edges_%s.csv' % graph_name, 'a')
while steps <= max_steps:
cost = pheromone_cost(G)
max_cost = max(max_cost, cost)
if cost_plot:
costs.append(cost)
prun_write_items = [steps, cost]
if curr_path_entropy != None:
prun_write_items.append(curr_path_entropy)
else:
prun_write_items.append('')
if curr_walk_entropy != None:
prun_write_items.append(curr_walk_entropy)
else:
prun_write_items.append('')
prun_str = ', '.join(map(str, prun_write_items))
if critical_edges_file != None:
w1 = 0
w2 = 0
critical_node = G.graph['critical_node']
critical_edge = G.graph['critical_edge']
critical_node_prev = G.graph['critical_node_prev']
for v in G.neighbors(critical_node):
if v == critical_node_prev:
continue
wt = G[critical_node][v]['weight']
if Ninv[(critical_node, v)] == Ninv[critical_edge]:
w1 += wt
else:
w2 += wt
critical_str = '%d, %f, %f\n' % (steps, w1, w2)
critical_edges_file.write(critical_str)
G2 = G.copy()
empty_nodes = set()
new_queue_nodes = set()
moved_ants = set()
empty_edges = set()
updated_paths_ants = set()
max_paths = maximal_paths(G, nests[0], nests[1])
max_path_lengths = []
for path in max_paths:
path_prob = path_prob_no_explore(G, path, strategy)
if path_prob > 0:
max_path_lengths.append(len(path))
mean_path_len = None
if len(max_path_lengths) > 0:
mean_path_len = mean(max_path_lengths)
if mean_path_len != None:
if max_path_len == None:
max_path_len = mean_path_len
else:
max_path_len = max(max_path_len, mean_path_len)
if DEBUG_QUEUES:
check_queues(G2, queued_nodes, queued_edges, num_ants)
for queued_node in queued_nodes:
curr_max_cycles = len(maximal_cycles(G, queued_node))
max_cycles = max(curr_max_cycles, max_cycles)
queue = G2.node[queued_node]['queue']
#queue = G2.node[node]['queue']
#assert len(queue) > 0
qlim = node_queue_lim
if qlim == -1:
qlim = len(queue)
next_ants = []
q = 0
while q < len(queue) and len(next_ants) < qlim:
if queue[q] not in moved_ants:
q_ant = queue[q]
next_ants.append(q_ant)
q += 1
for next_ant in next_ants:
queue.remove(next_ant)
for queued_ant in queue:
if queued_ant not in moved_ants:
moved_ants.add(queued_ant)
if video:
paths[queued_ant].append(queued_node)
if DEBUG_PATHS:
check_path(G, paths[queued_ant])
if track_pruning:
walks[queued_ant].append(queued_node)
#path = paths[queued_ant]
if len(queue) == 0:
empty_nodes.add(queued_node)
for next_ant in next_ants:
moved_ants.add(next_ant)
curr = currs[next_ant]
prev = prevs[next_ant]
if curr != origins[next_ant] and (not search_mode[next_ant])\
and pheromone_dead_end(G, curr, prev):
search_mode[next_ant] = True
n = list(G.neighbors(curr))
if curr != prev and prev != None:
n.remove(prev)
if len(n) == 0:
deadend[next_ant] = (curr not in nests)
#print curr, deadend[j]
elif len(n) > 1:
deadend[next_ant] = False
if (prev == curr) or (curr == origins[next_ant] and not search_mode[next_ant]):
prev = None
exp_prob = explore_prob
if search_mode[next_ant]:
exp_prob = explore2
elif (curr == origins[next_ant] and not search_mode[next_ant]):
exp_prob = 0
next, ex = next_edge(G, curr, exp_prob, strategy, prev, \
destinations[next_ant], search_mode[next_ant],\
backtrack)
if ex:
queue_ant(G2, curr, next_ant)
if not deadend[next_ant]:
add_amount = 2 * pheromone_add
if 'plant' in G.node[curr] and 'plant' in G.node[next]:
if G.node[curr]['plant'] != G.node[next]['plant']:
add_amount *= 0.5
G2[curr][next]['weight'] += add_amount
if decay_type == 'linear':
G2[curr][next]['units'].append(add_amount)
nonzero_edges.add(Ninv[(curr, next)])
new_queue_nodes.add(curr)
empty_nodes.discard(curr)
prevs[next_ant] = curr
currs[next_ant] = curr
if video:
paths[next_ant].append(curr)
if track_pruning:
walks[next_ant].append(curr)
if DEBUG_PATHS:
check_path(G, paths[next_ant])
else:
if (curr, next) == G[curr][next]['forwards']:
G2[curr][next]['forwards_queue'].append(next_ant)
else:
G2[curr][next]['backwards_queue'].append(next_ant)
queued_edges.add(Ninv[(curr, next)])
queued_nodes.difference_update(empty_nodes)
queued_nodes.update(new_queue_nodes)
if DEBUG_QUEUES:
check_queues(G2, queued_nodes, queued_edges, num_ants)
for edge_id in queued_edges:
u, v = N[edge_id]
resulting_size = 0
for direction in ['forwards', 'backwards']:
i = 0
curr, next = G2[u][v][direction]
if CRITICAL_NODES and 'critical_node' in G.graph:
if G.graph['critical_node'] in (u, v):
critical_nodes_file.write('%s, %s, %d\n' % (curr, next, steps))
edge_queue = G2[u][v][direction + '_queue']
eqlim = edge_queue_lim
if edge_queue_lim == -1:
eqlim = len(edge_queue)
while len(edge_queue) > 0 and i < eqlim:
next_ant = edge_queue.pop(0)
i += 1
queue_ant(G2, next, next_ant)
new_queue_nodes.add(next)
empty_nodes.discard(next)
prevs[next_ant] = curr
currs[next_ant] = next
if not deadend[next_ant]:
add_amount = pheromone_add
if 'plant' in G.node[curr] and 'plant' in G.node[next]:
if G.node[curr]['plant'] != G.node[next]['plant']:
add_amount *= 0.5
G2[curr][next]['weight'] += add_amount
if decay_type == 'linear':
G2[curr][next]['units'].append(add_amount)
nonzero_edges.add(Ninv[(curr, next)])
if video:
paths[next_ant].append(next)
if track_pruning:
walks[next_ant].append(next)
if DEBUG_PATHS:
check_path(G, paths[next_ant])
if next == destinations[next_ant]:
orig, dest = origins[next_ant], destinations[next_ant]
dest = next_destination(orig)
origins[next_ant], destinations[next_ant] = dest, orig
search_mode[next_ant] = False
if track_pruning :
walk = walks[next_ant]
if walk[0] == orig and walk[-1] == dest:
chosen_walk_counts[tuple(walk)] += 1
walk2 = remove_self_loops(walk)
path, cycle_count = walk_to_path(walk2)
start = path[0]
end = path[-1]
idx1 = nests.index(orig)
idx2 = nests.index(dest)
if idx2 > idx1:
path = path[::-1]
path_counts[tuple(path)] += 1
curr_path_entropy = entropy(path_counts.values())
curr_walk_entropy = entropy(chosen_walk_counts.values())
if max_entropy == None:
max_entropy = curr_path_entropy
else:
max_entropy = max(max_entropy, curr_path_entropy)
if max_walk_entropy == None:
max_walk_entropy = curr_walk_entropy
else:
max_walk_entropy = max(max_walk_entropy, curr_walk_entropy)
mean_chosen_path_len = weighted_mean_path_len(path_counts)
if max_chosen_path_len == None:
max_chosen_path_len = mean_chosen_path_len
else:
max_chosen_path_len = max(max_chosen_path_len, mean_chosen_path_len)
chosen_cycle_counts.append(cycle_count)
mean_chosen_cycle_count = mean(chosen_cycle_counts)
if max_chosen_cycle_count == None:
max_chosen_cycle_count = mean_chosen_cycle_count
else:
max_chosen_cycle_count = max(max_chosen_cycle_count, mean_chosen_cycle_count)
walks[next_ant] = [origins[next_ant]]
elif next == origins[next_ant]:
search_mode[next_ant] = True
for j in xrange(len(edge_queue)):
next_ant = edge_queue[j]
if video:
paths[next_ant].append(curr)
if DEBUG_PATHS:
check_path(G, paths[next_ant])
walks[next_ant].append(curr)
resulting_size += 1
if resulting_size == 0:
empty_edges.add(edge_id)
queued_nodes.difference_update(empty_nodes)
queued_nodes.update(new_queue_nodes)
queued_edges.difference_update(empty_edges)
if DEBUG_QUEUES:
check_queues(G2, queued_nodes, queued_edges, num_ants)
decay_func = get_decay_func_edges(decay_type)
zero_edges = decay_func(G2, nonzero_edges, pheromone_decay, time=1, min_pheromone=MIN_PHEROMONE)
nonzero_edges.difference_update(zero_edges)
G = G2
for u, v in sorted(G.edges()):
index = Ninv[(u, v)]
wt = G[u][v]['weight']
#unique_weights.add(wt)
max_weight = max(max_weight, wt)
if video:
if wt <= MIN_DETECTABLE_PHEROMONE:
edge_weights[index].append(None)
else:
edge_weights[index].append(wt)
if connect_time == -1 and has_pheromone_path(G, nests[0], nests[1]):
connect_time = steps
steps += 1
cost = 0
max_wt = 0
for u, v in G.edges():
wt = G[u][v]['weight']
if wt > MIN_PHEROMONE:
cost += 1
max_wt = max(wt, max_wt)
e_colors = edge_color[:]
e_widths = edge_width[:]
n_colors = node_color[:]
n_sizes = node_size[:]
for nest in nests:
n_colors[Minv[nest]] = 'm'
n_sizes[Minv[nest]] = 100
def init():
nx.draw(G, pos=pos, with_labels=False, node_size=n_sizes, edge_color=e_colors,\
node_color=n_colors, width=e_widths, nodelist = sorted(G.nodes()), \
edgelist = sorted(G.edges()))
def redraw(frame):
PP.clf()
print frame
n_colors = ['r'] * len(node_color)
n_sizes = [10] * len(node_size)
ax = PP.gca()
for n in xrange(num_ants):
node = paths[n][frame + 1]
index = Minv[node]
n_colors[index] = 'k'
n_sizes[index] += ant_thickness
#if frame > 0:
# frame -= 1
max_units = max_weight / pheromone_add
e_colors = []
e_widths = []
for u, v in sorted(G.edges()):
index = Ninv[(u, v)]
edge_wt = edge_weights[index][frame]
if edge_wt == None:
e_colors.append('k')
e_widths.append(1)
else:
e_colors.append('g')
e_widths.append(1 + 25 * (edge_wt / max_weight))
for nest in nests:
n_colors[Minv[nest]] = 'm'
#n_sizes[Minv[nest]] = min(n_sizes[Minv[nest]], 100)
#print nest, n_sizes[Minv[nest]]
nx.draw(G, pos=pos, with_labels=False, node_size=n_sizes, edge_color=e_colors, \
node_color=n_colors, width=e_widths, nodelist = sorted(G.nodes()), \
edgelist = sorted(G.edges()))
f = PP.draw()
return f,
if nframes == -1:
nframes = steps
if video:
ani = animation.FuncAnimation(fig, redraw, init_func=init, frames=nframes, \
interval = FRAME_INTERVAL)
mywriter = animation.AVConvWriter()
ani.save("ant_" + out_str + str(iter) + ".mp4", writer=mywriter)
if print_graph:
color_graph(G, 'g', (pheromone_add / max_wt), "graph_after_%s%d_e%0.2fd%0.2f" \
% (out_str, max_steps, explore_prob, pheromone_decay), cost)
print "graph colored"
costs.append(cost)
max_cost = max(max_cost, cost)
costs = PP.array(costs)
cost_pruning = (max_cost - cost) / float(max_cost)
if cost_plot:
figname = "pruning/pruning_%s%d_e%0.2fd%0.2f" % (out_str, max_steps, \
explore_prob, pheromone_decay, cost)
pruning_plot(costs, figname, max_cost)
path_pruning = None
if len(path_counts) > 0:
curr_path_entropy = entropy(path_counts.values())
max_entropy = max(max_entropy, curr_path_entropy)
path_pruning = max_entropy - curr_path_entropy
walk_pruning = None
if len(chosen_walk_counts) > 0:
curr_walk_entropy = entropy(chosen_walk_counts.values())
max_walk_entropy = max(max_walk_entropy, curr_walk_entropy)
walk_pruning = max_walk_entropy - curr_walk_entropy
# Output results.
path_lengths, revisits = [], []
right_nest, wrong_nest = 0.0, 0.0
hit_counts, miss_counts = [], []
nest, target = nests[0], nests[1]
has_path = has_pheromone_path(G, nest, target)
after_paths = []
if has_path:
if 'uniform' in strategy and UNIFORM_ENTROPY:
after_paths = pheromone_paths(G, nest, target, MAX_PATH_LENGTH)
else:
after_paths = maximal_paths(pheromone_subgraph(G), nest, target)
path_probs = []
useful_edges = set()
path_lenghts = []
for path in after_paths:
path_prob = path_prob_no_explore(G, path, strategy)
if path_prob > 0:
path_probs.append(path_prob)
edges = path_to_edges(path)
useful_edges.update(edges)
path_lengths.append(len(path))
wasted_edge_count, wasted_edge_weight = wasted_edges(G, useful_edges)
#print sorted(path_probs)
#print sum(path_probs)
path_etr = None
if len(path_probs) > 0:
path_etr = entropy(path_probs)
path_etr = np.abs(path_etr)
else:
has_path = False
mean_path_len = None
if len(path_lengths) > 0:
#mean_path_len = mean(path_lengths)
mean_path_len = PP.average(path_lengths, weights=path_probs)
if 'uniform' in strategy:
print "weighted mean path len", mean_path_len
mean_chosen_path_len = weighted_mean_path_len(path_counts)
mean_chosen_cycle_count = None
if len(chosen_cycle_counts) > 1:
mean_chosen_cycle_count = mean(chosen_cycle_counts)
path_len_pruning = None
if mean_path_len != None:
if max_path_len != None:
path_len_pruning = max_path_len - mean_path_len
chosen_path_len_pruning = None
if mean_chosen_path_len != None:
if max_chosen_path_len != None:
chosen_path_len_pruning = max_chosen_path_len - mean_chosen_path_len
chosen_cycle_count_pruning = None
if mean_chosen_cycle_count != None:
if max_chosen_cycle_count != None:
chosen_cycle_count_pruning = max_chosen_cycle_count - mean_chosen_cycle_count
print "has path", has_path
print "path entropy", path_etr
journey_times = []
journey_lengths = []
walk_counts = defaultdict(int)
total_steps = 0
cycles_pruning = None
if max_cycles > 0:
cycles_pruning = max_cycles - curr_max_cycles
header_items = []
write_items = [int(has_path), cost]
if (path_etr != None) and path_etr != float("-inf") and (not PP.isnan(path_etr)):
assert path_etr >= 0
write_items.append(path_etr)
else:
write_items.append('')
if len(walk_counts.values()) > 0:
write_items += [walk_entropy, mean_journey_time, med_journey_time, walk_success_rate]
else:
write_items += ['', '', '', '']
#write_items.append(walk_success_rate)
write_items.append(cost_pruning)
if connect_time != -1:
write_items.append(connect_time)
else:
write_items.append('')
if path_pruning != None:
write_items.append(path_pruning)
else:
write_items.append('')
if curr_path_entropy != None:
write_items.append(curr_path_entropy)
else:
write_items.append('')
if walk_pruning != None:
write_items.append(walk_pruning)
else:
write_items.append('')
if curr_walk_entropy != None:
write_items.append(curr_walk_entropy)
else:
write_items.append('')
write_items += [wasted_edge_count, wasted_edge_weight]
if mean_path_len != None:
write_items.append(mean_path_len)
else:
write_items.append('')
if cycles_pruning != None:
write_items.append(cycles_pruning)
else:
write_items.append('')
write_items.append(curr_max_cycles)
if path_len_pruning != None:
write_items.append(path_len_pruning)
else:
write_items.append('')
if chosen_path_len_pruning != None:
write_items.append(chosen_path_len_pruning)
else:
write_items.append('')
if chosen_cycle_count_pruning != None:
write_items.append(chosen_cycle_count_pruning)
else:
write_items.append('')
ant_str = ', '.join(map(str, write_items))
line = pher_str + ant_str + '\n'
data_file.write(line)
data_file.close()
pruning_file.close()
critical_nodes_file.close()
def repair(G, pheromone_add, pheromone_decay, explore_prob, strategy='rank',\
num_ants=100, max_steps=1000, num_iters=1, print_graph=False, video=False, \
nframes=-1, cost_plot=False, backtrack=False, \
decay_type='exp', node_queue_lim=1, edge_queue_lim=1, one_way=False):
graph_name = G.graph['name']
nests = G.graph['nests']
out_items = ['repair', strategy, graph_name, decay_type]
if backtrack:
out_items.append('backtrack')
if one_way:
out_items.append('one_way')
out_str = '_'.join(out_items)
savedir = '%s/%s/%s' % (graph_name, strategy, decay_type)
header_items = ['graph', 'strategy', 'decay type', 'ants', 'max steps',\
'backtrack', 'one_way', 'node queue lim', 'edge queue lim',\
'explore', 'decay']
write_items = [graph_name, strategy, decay_type, num_ants, max_steps,\
backtrack, one_way, node_queue_lim, edge_queue_lim,\
explore_prob, pheromone_decay]
def next_destination(origin):
idx = nests.index(origin)
idx += 1
idx %= len(nests)
return nests[idx]
num_edges = G.size()
nframes = min(nframes, max_steps)
data_fname = '/iblsn/data/Arjun/Ants/ant_repair.csv'
first_time = not os.path.exists(data_fname)
if video:
fig = PP.figure()
init_path = G.graph['init path']
track_pruning = max_steps <= MAX_PRUNING_STEPS
for iter in xrange(num_iters):
nonzero_edges = set()
for u, v in G.edges():
G[u][v]['weight'] = MIN_PHEROMONE
G[u][v]['units'] = []
for u, v in init_path:
G[u][v]['weight'] += pheromone_add * INIT_WEIGHT_FACTOR
if decay_type == 'linear':
G[u][v]['units']+= [pheromone_add] * INIT_WEIGHT_FACTOR
nonzero_edges.add(Ninv[(u, v)])
clear_queues(G)
explore = defaultdict(bool)
prevs = {}
currs = {}
paths = {}
walks = {}
destinations = {}
origins = {}
edge_weights = defaultdict(list)
deadend = {}
search_mode = defaultdict(lambda: False)
costs = []
path_counts = defaultdict(int)
chosen_walk_counts = defaultdict(int)
max_entropy = None
max_walk_entropy = None
connect_time = -1
queued_nodes = set()
queued_edges = set()
max_cycles = 0
curr_max_cycles = 0
max_path_len = None
max_chosen_path_len = None
chosen_cycle_counts = []
max_chosen_cycle_count = None
curr_ants = num_ants
if curr_ants == -1:
curr_ants = max_steps // 10
for ant in xrange(curr_ants):
origin = nests[ant % len(nests)]
origins[ant] = origin
destinations[ant] = next_destination(origin)
prev, curr = None, None
if one_way:
prev, curr = init_path[choice((len(init_path) // 2))]
else:
prev, curr = init_path[choice(len(init_path))]
if random() <= 0.5:
prev, curr = curr, prev
paths[ant] = [prev, curr]
if destinations[ant] in paths[ant]:
origins[ant], destinations[ant] = destinations[ant], origins[ant]
walks[ant] = [prev, curr]
prevs[ant] = prev
currs[ant] = curr
deadend[ant] = False
queue_ant(G, curr, ant)
queued_nodes.add(curr)
steps = 1
rounds = 1
max_weight = MIN_PHEROMONE
unique_weights = set()
max_cost = 0
costs = []
curr_path_entropy = None
curr_walk_entropy = None
for u, v in sorted(G.edges()):
index = Ninv[(u, v)]
wt = G[u][v]['weight']
#unique_weights.add(wt)
max_weight = max(max_weight, wt)
if video:
if wt <= MIN_DETECTABLE_PHEROMONE:
edge_weights[index].append(None)
else:
edge_weights[index].append(wt)
while steps <= max_steps:
cost = pheromone_cost(G)
max_cost = max(max_cost, cost)
G2 = G.copy()
empty_nodes = set()
new_queue_nodes = set()
moved_ants = set()
empty_edges = set()
updated_paths_ants = set()
max_paths = maximal_paths(G, nests[0], nests[1])
max_path_lengths = []
for path in max_paths:
path_prob = path_prob_no_explore(G, path, strategy)
if path_prob > 0:
max_path_lengths.append(len(path))
mean_path_len = None
if len(max_path_lengths) > 0:
mean_path_len = mean(max_path_lengths)
if mean_path_len != None:
if max_path_len == None:
max_path_len = mean_path_len
else:
max_path_len = max(max_path_len, mean_path_len)
if DEBUG_QUEUES:
check_queues(G2, queued_nodes, queued_edges, curr_ants)
for queued_node in queued_nodes:
curr_max_cycles = len(maximal_cycles(G, queued_node))
max_cycles = max(curr_max_cycles, max_cycles)
queue = G2.node[queued_node]['queue']
#queue = G2.node[node]['queue']
#assert len(queue) > 0
qlim = node_queue_lim
if qlim == -1:
qlim = len(queue)
next_ants = []
q = 0
while q < len(queue) and len(next_ants) < qlim:
if queue[q] not in moved_ants:
q_ant = queue[q]
next_ants.append(q_ant)
q += 1
for next_ant in next_ants:
queue.remove(next_ant)
for queued_ant in queue:
if queued_ant not in moved_ants:
moved_ants.add(queued_ant)
if video:
paths[queued_ant].append(queued_node)
if DEBUG_PATHS:
check_path(G, paths[queued_ant])
if track_pruning:
walks[queued_ant].append(queued_node)
if len(queue) == 0:
empty_nodes.add(queued_node)
for next_ant in next_ants:
moved_ants.add(next_ant)
curr = currs[next_ant]
prev = prevs[next_ant]
if curr != origins[next_ant] and (not search_mode[next_ant])\
and pheromone_dead_end(G, curr, prev):
search_mode[next_ant] = True
n = list(G.neighbors(curr))
if curr != prev and prev != None:
n.remove(prev)
if len(n) == 0:
deadend[next_ant] = (curr not in nests)
#print curr, deadend[j]
elif len(n) > 1:
deadend[next_ant] = False
if (prev == curr) or (curr == origins[next_ant] and not search_mode[next_ant]):
prev = None
exp_prob = explore_prob
if (curr == origins[next_ant] and not search_mode[next_ant]):
exp_prob = 0
next, ex = next_edge(G, curr, exp_prob, strategy, prev, \
destinations[next_ant], search_mode[next_ant],\
backtrack)
if G[curr][next]['weight'] <= MIN_DETECTABLE_PHEROMONE:
queue_ant(G2, curr, next_ant)
if not deadend[next_ant]:
add_amount = 2 * pheromone_add
G2[curr][next]['weight'] += add_amount
if decay_type == 'linear':
G2[curr][next]['units'].append(add_amount)
nonzero_edges.add(Ninv[(curr, next)])
new_queue_nodes.add(curr)
empty_nodes.discard(curr)
prevs[next_ant] = next
currs[next_ant] = curr
if video:
paths[next_ant].append(curr)
if track_pruning:
walks[next_ant].append(curr)
if DEBUG_PATHS:
check_path(G, paths[next_ant])
else:
if (curr, next) == G[curr][next]['forwards']:
G2[curr][next]['forwards_queue'].append(next_ant)
else:
G2[curr][next]['backwards_queue'].append(next_ant)
queued_edges.add(Ninv[(curr, next)])
queued_nodes.difference_update(empty_nodes)
queued_nodes.update(new_queue_nodes)
if DEBUG_QUEUES:
check_queues(G2, queued_nodes, queued_edges, curr_ants)
for edge_id in queued_edges:
u, v = N[edge_id]
resulting_size = 0
for direction in ['forwards', 'backwards']:
i = 0
curr, next = G2[u][v][direction]
if CRITICAL_NODES and 'critical_node' in G.graph:
if G.graph['critical_node'] in (u, v):
critical_nodes_file.write('%s, %s, %d\n' % (curr, next, steps))
edge_queue = G2[u][v][direction + '_queue']
eqlim = edge_queue_lim
if edge_queue_lim == -1:
eqlim = len(edge_queue)
while len(edge_queue) > 0 and i < eqlim:
next_ant = edge_queue.pop(0)
i += 1
queue_ant(G2, next, next_ant)
new_queue_nodes.add(next)
empty_nodes.discard(next)
prevs[next_ant] = curr
currs[next_ant] = next
if not deadend[next_ant]:
add_amount = pheromone_add
if 'plant' in G.node[curr] and 'plant' in G.node[next]:
if G.node[curr]['plant'] != G.node[next]['plant']:
add_amount *= 0.5
G2[curr][next]['weight'] += add_amount
if decay_type == 'linear':
G2[curr][next]['units'].append(add_amount)
nonzero_edges.add(Ninv[(curr, next)])
if video:
paths[next_ant].append(next)
if track_pruning:
walks[next_ant].append(next)
if DEBUG_PATHS:
check_path(G, paths[next_ant])
if next == destinations[next_ant]:
orig, dest = origins[next_ant], destinations[next_ant]
dest = next_destination(orig)
origins[next_ant], destinations[next_ant] = dest, orig
search_mode[next_ant] = False
if track_pruning :
walk = walks[next_ant]
if walk[0] == orig and walk[-1] == dest:
chosen_walk_counts[tuple(walk)] += 1
walk2 = remove_self_loops(walk)
path, cycle_count = walk_to_path(walk2)
start = path[0]
end = path[-1]
idx1 = nests.index(orig)
idx2 = nests.index(dest)
if idx2 > idx1:
path = path[::-1]
path_counts[tuple(path)] += 1
curr_path_entropy = entropy(path_counts.values())
curr_walk_entropy = entropy(chosen_walk_counts.values())
if max_entropy == None:
max_entropy = curr_path_entropy
else:
max_entropy = max(max_entropy, curr_path_entropy)
if max_walk_entropy == None:
max_walk_entropy = curr_walk_entropy
else:
max_walk_entropy = max(max_walk_entropy, curr_walk_entropy)
mean_chosen_path_len = weighted_mean_path_len(path_counts)
if max_chosen_path_len == None:
max_chosen_path_len = mean_chosen_path_len
else:
max_chosen_path_len = max(max_chosen_path_len, mean_chosen_path_len)
chosen_cycle_counts.append(cycle_count)
mean_chosen_cycle_count = mean(chosen_cycle_counts)
if max_chosen_cycle_count == None:
max_chosen_cycle_count = mean_chosen_cycle_count
else:
max_chosen_cycle_count = max(max_chosen_cycle_count, mean_chosen_cycle_count)
walks[next_ant] = [origins[next_ant]]
elif next == origins[next_ant]:
search_mode[next_ant] = True
for j in xrange(len(edge_queue)):
next_ant = edge_queue[j]
if video:
paths[next_ant].append(curr)
if DEBUG_PATHS:
check_path(G, paths[next_ant])
walks[next_ant].append(curr)
resulting_size += 1
if resulting_size == 0:
empty_edges.add(edge_id)
queued_nodes.difference_update(empty_nodes)
queued_nodes.update(new_queue_nodes)
queued_edges.difference_update(empty_edges)
if DEBUG_QUEUES:
check_queues(G2, queued_nodes, queued_edges, curr_ants)
decay_func = get_decay_func_edges(decay_type)
zero_edges = decay_func(G2, nonzero_edges, pheromone_decay,
time=SECONDS_PER_STEP, min_pheromone=MIN_PHEROMONE)
nonzero_edges.difference_update(zero_edges)
if num_ants == -1:
for nest in nests:
ant = curr_ants
curr = nest
origin = nest
origins[ant] = origin
destinations[ant] = next_destination(origin)
if video:
paths[ant] = ([None] * (steps + 1)) + [curr]
if destinations[ant] == curr:
origins[ant], destinations[ant] = destinations[ant], origins[ant]
walks[ant] = [curr]
prevs[ant] = None
currs[ant] = curr
deadend[ant] = False
queue_ant(G2, curr, ant)
queued_nodes.add(curr)
curr_ants += 1
G = G2
for u, v in sorted(G.edges()):
index = Ninv[(u, v)]
wt = G[u][v]['weight']
#unique_weights.add(wt)
max_weight = max(max_weight, wt)
if video:
if wt <= MIN_DETECTABLE_PHEROMONE:
edge_weights[index].append(None)
else:
edge_weights[index].append(wt)
if connect_time == -1 and has_pheromone_path(G, nests[0], nests[1]):
connect_time = steps
steps += 1
cost = 0
max_wt = 0
for u, v in G.edges():
wt = G[u][v]['weight']
if wt > MIN_PHEROMONE:
cost += 1
max_wt = max(wt, max_wt)
e_colors = edge_color[:]
e_widths = edge_width[:]
n_colors = node_color[:]
n_sizes = node_size[:]
for nest in nests:
n_colors[Minv[nest]] = 'm'
n_sizes[Minv[nest]] = 100
def init():
nx.draw(G, pos=pos, with_labels=False, node_size=n_sizes, edge_color=e_colors,\
node_color=n_colors, width=e_widths, nodelist = sorted(G.nodes()), \
edgelist = sorted(G.edges()))
def redraw(frame):
PP.clf()
print frame
n_colors = ['r'] * len(node_color)
n_sizes = [10] * len(node_size)
ax = PP.gca()
for p in paths:
node = paths[p][frame + 1]
if node != None:
index = Minv[node]
n_colors[index] = 'k'
n_sizes[index] += ant_thickness
max_units = max_weight / pheromone_add
e_colors = []
e_widths = []
for u, v in sorted(G.edges()):
index = Ninv[(u, v)]
edge_wt = edge_weights[index][frame]
if edge_wt == None:
e_colors.append('k')
e_widths.append(1)
else:
e_colors.append('g')
e_widths.append(1 + 25 * (edge_wt / max_weight))
for nest in nests:
n_colors[Minv[nest]] = 'm'
nx.draw(G, pos=pos, with_labels=False, node_size=n_sizes, edge_color=e_colors, \
node_color=n_colors, width=e_widths, nodelist = sorted(G.nodes()), \
edgelist = sorted(G.edges()))
f = PP.draw()
return f,
if nframes == -1:
nframes = steps
if video:
ani = animation.FuncAnimation(fig, redraw, init_func=init, frames=nframes, \
interval = FRAME_INTERVAL)
vid_dir = 'figs/videos/%s' % savedir
os.system('mkdir -p ' + vid_dir)
ani.save('%s/ant_%s%d.mp4' % (vid_dir, out_str, iter), writer='avconv')
return None
if print_graph:
outdir = 'figs/after_graphs/' + savedir
os.system('mkdir -p ' + outdir)
figname = outdir + '/graph_after_%s%d_e%0.2fd%0.2f' % (out_str, max_steps, explore_prob, pheromone_decay)
color_graph(G, 'g', (pheromone_add / max_wt), figname, cost)
print "graph colored"
costs.append(cost)
max_cost = max(max_cost, cost)
costs = PP.array(costs)
cost_pruning = (max_cost - cost) / float(max_cost)
if cost_plot:
outdir = 'figs/cost_plots/' + savedir
os.system('mkdir -p ' + outdir)
figname = outdir + "/cost_plot_%s%d_e%0.2fd%0.2f" %\
(savedir, out_str, max_steps, explore_prob,\
pheromone_decay, cost)
pruning_plot(costs, figname, max_cost)
path_pruning = None
if len(path_counts) > 0:
curr_path_entropy = entropy(path_counts.values())
max_entropy = max(max_entropy, curr_path_entropy)
path_pruning = max_entropy - curr_path_entropy
walk_pruning = None
if len(chosen_walk_counts) > 0:
curr_walk_entropy = entropy(chosen_walk_counts.values())
max_walk_entropy = max(max_walk_entropy, curr_walk_entropy)
walk_pruning = max_walk_entropy - curr_walk_entropy
# Output results.
path_lengths, revisits = [], []
right_nest, wrong_nest = 0.0, 0.0
hit_counts, miss_counts = [], []
nest, target = nests[0], nests[1]
has_path = has_pheromone_path(G, nest, target)
after_paths = []
if has_path:
if False:#'uniform' in strategy:
after_paths = pheromone_paths(G, nest, target, MAX_PATH_LENGTH)
else:
after_paths = maximal_paths(pheromone_subgraph(G), nest, target)
path_probs = []
useful_edges = set()
path_lenghts = []
for path in after_paths:
path_prob = path_prob_no_explore(G, path, strategy)
if path_prob > 0:
path_probs.append(path_prob)
edges = path_to_edges(path)
useful_edges.update(edges)
path_lengths.append(len(path))
wasted_edge_count, wasted_edge_weight = wasted_edges(G, useful_edges)
path_etr = None
if len(path_probs) > 0:
path_etr = entropy(path_probs)
path_etr = np.abs(path_etr)
else:
has_path = False
mean_path_len = None
if len(path_lengths) > 0:
mean_path_len = mean(path_lengths)
mean_chosen_path_len = weighted_mean_path_len(path_counts)
mean_chosen_cycle_count = None
if len(chosen_cycle_counts) > 1:
mean_chosen_cycle_count = mean(chosen_cycle_counts)
path_len_pruning = None
if mean_path_len != None:
if max_path_len != None:
path_len_pruning = max_path_len - mean_path_len
chosen_path_len_pruning = None
if mean_chosen_path_len != None:
if max_chosen_path_len != None:
chosen_path_len_pruning = max_chosen_path_len - mean_chosen_path_len
chosen_cycle_count_pruning = None
if mean_chosen_cycle_count != None:
if max_chosen_cycle_count != None:
chosen_cycle_count_pruning = max_chosen_cycle_count - mean_chosen_cycle_count
print "has path", has_path
print "path entropy", path_etr
journey_times = []
journey_lengths = []
walk_counts = defaultdict(int)
total_steps = 0
cycles_pruning = None
if max_cycles > 0:
cycles_pruning = max_cycles - curr_max_cycles
header_items += ['has path', 'graph cost']
write_items += [int(has_path), cost]
header_items.append('path entropy')
if (path_etr != None) and path_etr != float("-inf") and (not PP.isnan(path_etr)):
assert path_etr >= 0
write_items.append(path_etr)
else:
write_items.append('')
header_items.append('cost pruning')
write_items.append(cost_pruning)
header_items.append('connect time')
if connect_time != -1:
write_items.append(connect_time)
else:
write_items.append('')
header_items.append('path pruning')
if path_pruning != None:
write_items.append(path_pruning)
else:
write_items.append('')
header_items.append('current path entropy')
if curr_path_entropy != None:
write_items.append(curr_path_entropy)
else:
write_items.append('')
header_items.append('walk pruning')
if walk_pruning != None:
write_items.append(walk_pruning)
else:
write_items.append('')
header_items.append('current walk entropy')
if curr_walk_entropy != None:
write_items.append(curr_walk_entropy)
else:
write_items.append('')
header_items += ['wasted edge count', 'wasted edge weight']
write_items += [wasted_edge_count, wasted_edge_weight]
header_items.append('mean path length')
if mean_path_len != None:
write_items.append(mean_path_len)
else:
write_items.append('')
header_items.append('cycles pruning')
if cycles_pruning != None:
write_items.append(cycles_pruning)
else:
write_items.append('')
header_items.append('current max cycles')
write_items.append(curr_max_cycles)
header_items.append('path length pruning')
if path_len_pruning != None:
write_items.append(path_len_pruning)
else:
write_items.append('')
header_items.append('chosen path length pruning')
if chosen_path_len_pruning != None:
write_items.append(chosen_path_len_pruning)
else:
write_items.append('')
header_items.append('chosen cycle count pruning')
if chosen_cycle_count_pruning != None:
write_items.append(chosen_cycle_count_pruning)
else:
write_items.append('')
with open(data_fname, 'a') as data_file:
header_str = ', '.join(map(str, header_items))
if first_time:
data_file.write(header_str + '\n')
data_str = ', '.join(map(str, write_items))
data_file.write(line + '\n')
rgax[:,11] = rgax[:,11] + dmag rg[pl.find(rg[:,8] < 0),8] = rg[pl.find(rg[:,8] < 0),8] + 360 rg[pl.find(rg[:,13] < 0),13] = rg[pl.find(rg[:,13] < 0),13] + 360 rg[pl.find(rg[:,16] < 0),16] = rg[pl.find(rg[:,16] < 0),16] + 360 rgax[pl.find(rgax[:,11] < 0),11] = rgax[pl.find(rgax[:,11] < 0),11] + 360 #datas # dre = np.array([datetime.strptime(str(int(re[i,0])), '%Y%m%d%H%M') for i in range(len(re))]) dsa = np.array([datetime.strptime(str(int(sa[i,0])), '%Y%m%d%H%M') for i in range(len(sa))]) dfl = np.array([datetime.strptime(str(int(fl[i,0])), '%Y%m%d%H%M') for i in range(len(fl))]) drg = np.array([datetime.strptime(str(int(rg[i,0])), '%Y%m%d%H%M') for i in range(len(rg))]) drgax = np.array([datetime.strptime(rgax_data[i,0]+'-'+rgax_data[i,1], '%Y/%m/%d-%H:%M') for i in range(len(rgax))]) #retira os valores com nan # re1 = re[pl.find(pl.isnan(re[:,1])==False),:] sa1 = sa[pl.find(pl.isnan(sa[:,1])==False),:] fl1 = fl[pl.find(pl.isnan(fl[:,1])==False),:] rg1 = rg[pl.find(pl.isnan(rg[:,1])==False),:] #################################################################################### #################################################################################### #comparacao da serie temporal da axys (summary) e lioc #janela para plotagem com as 3 medicoes a1 = datetime(2012,8,19,1,0) a2 = datetime(2012,9,30,1,0) ms = 9 #markersize pl.figure()
def phot1(r,
imlist,
plot=True,
names=None,
panel=None,
debug=None,
showmask="both"):
"""
give me a region and an imlist and tell me whether to plot cutouts
"""
nim = len(imlist)
# TODO alg for how many subpanels.
if panel == None: panel = [5, 4, 1] # for vertical page
f = []
df = []
raw = []
for j in range(nim):
im = imlist[j]
# if plotting, need to make image cutouts; even if not, this tells
# us if the source is off the edge
xtents = r.imextents(imlist[j])
minsize = 5
if (xtents[3] - xtents[1]) < minsize or (xtents[2] -
xtents[0]) < minsize:
raw0, f0, df0 = 0, 0, 0
print "phot region too small - %f,%f less than %d pixels" % (
(xtents[3] - xtents[1]), (xtents[2] - xtents[0]), minsize)
print xtents, r.imcoords(im, reg="bg1")
else:
if plot:
pl.subplot(panel[0], panel[1], panel[2])
im = hextract(imlist[j], xtents)[0]
## ROTATE
rotate = True
if rotate:
from astropy import wcs
w = wcs.WCS(im.header)
from scipy.ndimage.interpolation import rotate
if w.wcs.has_crota():
t0 = w.wcs.crota[1]
else:
t0 = pl.arctan2(w.wcs.cd[0, 1],
-w.wcs.cd[0, 0]) * 180 / pl.pi
theta = -1 * t0
im.data = rotate(im.data, theta, reshape=False)
ct = pl.cos(pl.pi * theta / 180)
st = pl.sin(pl.pi * theta / 180)
if w.wcs.has_crota():
w.wcs.crota[1] = w.wcs.crota[1] + theta
im.header['CROTA2'] = w.wcs.crota[1]
print "rotating crota by " + str(theta)
else:
w.wcs.cd = pl.matrix(w.wcs.cd) * pl.matrix([[ct, -st],
[st, ct]])
im.header['CD1_1'] = w.wcs.cd[0, 0]
im.header['CD1_2'] = w.wcs.cd[0, 1]
im.header['CD2_1'] = w.wcs.cd[1, 0]
im.header['CD2_2'] = w.wcs.cd[1, 1]
print "rotating cd by " + str(theta)
#pdb.set_trace()
# ugh need minmax of aperture region...
# estimate as inner 1/2 for now
s = im.shape
z = im.data[int(s[0] * 0.25):int(s[0] * 0.75),
int(s[1] * 0.25):int(s[1] * 0.75)]
if len(z[0]) <= 0:
print z
z = pl.where(pl.isnan(im.data))
if len(z[0]) > 0:
z = pl.where(pl.isnan(im.data) == False)
std = im.data[z[0], z[1]].std()
else:
std = im.data.std()
rg = pl.median(im.data) + pl.array([-0.5, 5]) * std
# marta wants them less saturated
rg[1] = pl.nanmax(im.data)
if rg[0] < 0: rg[0] = 0
if rg[1] <= rg[0]:
rg = [pl.nanmin(z), pl.nanmax(z)]
if showmask == False or showmask == "both": # show the jet one
pl.imshow(im.data,
origin="bottom",
interpolation="nearest",
vmin=rg[0],
vmax=rg[1])
elif showmask == True: # only show the mask
pl.imshow(im.data,
origin="bottom",
interpolation="nearest",
vmin=rg[0],
vmax=rg[1],
cmap="YlGn")
ax = pl.gca()
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
if names:
pl.text(0.05,
0.99,
names[j],
horizontalalignment='left',
verticalalignment='top',
transform=ax.transAxes,
bbox=dict(facecolor='white', alpha=0.5))
pl.xlim([-0.5, s[1] - 0.5])
pl.ylim([-0.5, s[0] - 0.5])
if showmask == False:
r.plotimx(im) # overplot the apertures
if showmask == "both":
panel[2] = panel[2] + 1
pl.subplot(panel[0], panel[1], panel[2])
rg = pl.median(im.data) + pl.array([-0.5, 5]) * std
if rg[0] < 0: rg[0] = 0
if rg[1] <= rg[0]:
rg = [pl.nanmin(z), pl.nanmax(z)]
pl.imshow(im.data,
origin="bottom",
interpolation="nearest",
vmin=rg[0],
vmax=rg[1],
cmap="YlGn")
ax = pl.gca()
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
if names:
pl.text(0.05,
0.99,
names[j],
horizontalalignment='left',
verticalalignment='top',
transform=ax.transAxes,
bbox=dict(facecolor='white', alpha=0.5))
pl.xlim([-0.5, s[1] - 0.5])
pl.ylim([-0.5, s[0] - 0.5])
if debug:
# r.debug=True
if names:
print names[j]
raw0, bg, f0, df0 = r.phot(im)
raw.append(raw0)
f.append(f0)
df.append(df0)
panel[2] = panel[2] + 1
if plot:
pl.subplots_adjust(wspace=0.02,
hspace=0.02,
left=0.1,
right=0.97,
top=0.95,
bottom=0.05)
return f, df, raw
for idata in range(len(datam_pystr)):
nest[idata] = np.where(datam_ww3str == datam_pystr[idata])[0]
nest = nest.astype(int)
ww3 = ww3[nest]
datam_ww3 = datam_ww3[nest]
###########################################################################
#realiza estatisticas (bias, rmse, si, corr)
#seleciona valor sem nan
aux = pl.find(pl.isnan(py[:,6])==False)
hs = py[aux,6]
tp = py[aux,7]
dp = py[aux,8]
hs_mod = ww3[aux,5]
tp_mod = ww3[aux,6]
dp_mod = ww3[aux,7]
### BIAS ###
#hs
bias_hs = np.mean(hs_mod - hs)
#tp
bias_tp = np.mean(tp_mod - tp)
#dp