def find_short_path(aux_structures, loc1, loc2):
"""
Return the shortest path between the two locations
Parameters:
aux_structures: the result of calling build_auxiliary_structures
loc1: tuple of 2 floats: (latitude, longitude), representing the start
location
loc2: tuple of 2 floats: (latitude, longitude), representing the end
location
Returns:
a list of (latitude, longitude) tuples representing the shortest path
(in terms of distance) from loc1 to loc2.
"""
location = aux_structures[1]
final_path = []
distance_1 = float('inf')
distance_2 = float('inf')
for node in location:
distance_new = great_circle_distance(loc1, location[node])
distance_2_new = great_circle_distance(loc2, location[node])
if distance_new < distance_1:
node1 = node
distance_1 = distance_new
if distance_2_new < distance_2:
node2 = node
distance_2 = distance_2_new
path = find_short_path_nodes(aux_structures, node1, node2)
if path == None:
return None
for node in path:
final_path.append(location[node])
return final_path
def build_auxiliary_structures(nodes_filename, ways_filename):
"""
Create any auxiliary structures you are interested in, by reading the data
from the given filenames (using read_osm_data)
"""
#The datatype I will be a dict with keys of eligible node ids
#with values containing a set of tuples of adjacent eligible nodes:
#{node: {(id,lat,lon,distance,speed_limit)}}
answer_dict={}
big_nodes={}
#for way in read_osm_data(ways_filename):
# print (way)
for y in read_osm_data(nodes_filename):
big_nodes.update({y['id']:y})
for x in read_osm_data(ways_filename):
if 'highway' in x['tags'].keys():
if x['tags']['highway'] in ALLOWED_HIGHWAY_TYPES:
for w in range(len(x['nodes'])):
#find speed limit:
if 'maxspeed_mph' in x['tags'].keys():
speed=x['tags']['maxspeed_mph']
else:
speed=DEFAULT_SPEED_LIMIT_MPH[x['tags']['highway']]
#adds a new key if id is not in answer_dict
if x['nodes'][w] not in answer_dict.keys():
answer_dict.update({x['nodes'][w]:set()})
if w!=len(x['nodes'])-1:
#calculate distance between x['nodes'][w] and x['nodes'][w+1]
lat=big_nodes[x['nodes'][w+1]]['lat']
lon=big_nodes[x['nodes'][w+1]]['lon']
oth_lat=big_nodes[x['nodes'][w]]['lat']
oth_lon=big_nodes[x['nodes'][w]]['lon']
dist=great_circle_distance((lat,lon), (oth_lat,oth_lon))
answer_dict[x['nodes'][w]].add((x['nodes'][w+1],lat,lon,dist,speed))
if 'oneway' in x['tags'].keys():
if x['tags']['oneway']!='yes':
if w!=0:
#calculate distance between x['nodes'][w] and x['nodes'][w-1]
lat=big_nodes[x['nodes'][w-1]]['lat']
lon=big_nodes[x['nodes'][w-1]]['lon']
oth_lat=big_nodes[x['nodes'][w]]['lat']
oth_lon=big_nodes[x['nodes'][w]]['lon']
dist=great_circle_distance((lat,lon), (oth_lat,oth_lon))
answer_dict[x['nodes'][w]].add((x['nodes'][w-1],lat,lon,dist,speed))
else:
if w!=0:
#calculate distance between x['nodes'][w] and x['nodes'][w-1]
lat=big_nodes[x['nodes'][w-1]]['lat']
lon=big_nodes[x['nodes'][w-1]]['lon']
oth_lat=big_nodes[x['nodes'][w]]['lat']
oth_lon=big_nodes[x['nodes'][w]]['lon']
dist=great_circle_distance((lat,lon), (oth_lat,oth_lon))
answer_dict[x['nodes'][w]].add((x['nodes'][w-1],lat,lon,dist,speed))
#removing any empty keys
actual_dict={}
for f in answer_dict.keys():
if answer_dict[f]!=set():
actual_dict.update({f:answer_dict[f]})
return actual_dict
def find_fast_path(aux_structures, loc1, loc2):
"""
Return the shortest path between the two locations, in terms of expected
time (taking into account speed limits).
Parameters:
aux_structures: the result of calling build_auxiliary_structures
loc1: tuple of 2 floats: (latitude, longitude), representing the start
location
loc2: tuple of 2 floats: (latitude, longitude), representing the end
location
Returns:
a list of (latitude, longitude) tuples representing the shortest path
(in terms of time) from loc1 to loc2.
"""
location = aux_structures[1]
final_path = []
distance_1 = float('inf')
distance_2 = float('inf')
for node in location:
distance_new = great_circle_distance(loc1, location[node])
distance_2_new = great_circle_distance(loc2, location[node])
if distance_new < distance_1:
node1 = node
distance_1 = distance_new
if distance_2_new < distance_2:
node2 = node
distance_2 = distance_2_new
considered_paths = [([node1], 0, 0)]
previous_nodes = set()
node_path = []
while len(considered_paths) != 0:
considered_paths.sort(key=lambda tup: tup[1] + tup[2])
shortest_value = considered_paths.pop(0)
if shortest_value[0][-1] in previous_nodes:
continue
if shortest_value[0][-1] == node2:
node_path.extend(shortest_value[0])
break
else:
previous_nodes.add(shortest_value[0][-1])
for node in aux_structures[0].get(shortest_value[0][-1], {}):
cost = aux_structures[2][shortest_value[0][-1]][node]
if node in previous_nodes:
continue
else:
considered_paths.append((shortest_value[0] + [node],
shortest_value[1] + cost, 0))
if len(node_path) == 0:
return None
for node in node_path:
final_path.append(location[node])
return final_path
def find_path(aux_structures, loc1, loc2, short=True):
'''
Finds a path from loc1 to loc2.
'''
n1 = find_nearest_node(loc1, aux_structures)
n2 = find_nearest_node(loc2, aux_structures)
agenda = [[0, 0, [n1]]]
visited = set()
while agenda:
if short:
agenda = sorted(agenda, key=lambda x: x[0])
else:
agenda = sorted(agenda, key=lambda x: x[1])
popped = agenda.pop(0)
current_path = popped[2]
# create agenda by looping thru children of terminal node of current path
terminal_node = current_path[-1]
if terminal_node not in visited:
if terminal_node == n2: # if terminal node is destination node
return [aux_structures[1][node] for node in current_path]
visited.add(terminal_node)
children_of_terminal_n = aux_structures[0][terminal_node]
for neighbor in children_of_terminal_n: #loop thru neighbor nodes
if neighbor not in visited:
new_path = current_path.copy()
new_path.append(neighbor)
if short:
d = popped[1] #distance so far
d += great_circle_distance(
(
aux_structures[1][neighbor]
[0], #added distance of neighbor node from last node
aux_structures[1][neighbor][1]),
(aux_structures[1][terminal_node][0],
aux_structures[1][terminal_node][1]))
h = great_circle_distance(
aux_structures[1][neighbor],
aux_structures[1][n2]) # heuristic
agenda.append([h + d, d, new_path])
else:
t = popped[1]
t += great_circle_distance(
(aux_structures[1][neighbor][0],
aux_structures[1][neighbor][1]),
(aux_structures[1][terminal_node][0],
aux_structures[1][terminal_node][1]
)) / aux_structures[0][terminal_node][
neighbor] #finds time to new node
agenda.append([0, t, new_path])
return None
def find_path(graph, loc1, loc2, short):
def find_start_end(loc1, loc2):
"""
Find start/end nodes near loc1/loc2, by iterating over a graph
of all valid nodes (those appeared on some ways); and find smallest great_circle_distance
"""
start = end = None # start node, end node
min_start_distance = min_end_distance = float('inf')
for node_id in graph:
node_loc = graph[node_id]['loc']
distance_start = great_circle_distance(loc1, node_loc)
distance_end = great_circle_distance(loc2, node_loc)
if distance_start < min_start_distance:
start, min_start_distance = node_id, distance_start
if distance_end < min_end_distance:
end, min_end_distance = node_id, distance_end
return start, end
start, end = find_start_end(loc1, loc2)
# find shortest/fastest path from start to end, using uniform-cost search
agenda = [(0, [start])] # list of tuples (cost, path)
expanded = set()
# pulled_agenda = 0 ## counter, comparison of using heuristic vs not-using-heuristic
while agenda:
# find lowest cost path currently in agenda || would be nice if i am allowed to import heapq
# HEURISTIC
if short: # use heuristic
min_cost_so_far, min_index = float('inf'), None
for i, (cost_heuristic, path) in enumerate(agenda):
cost_heuristic += great_circle_distance(
graph[path[-1]]['loc'], graph[end]['loc'])
if cost_heuristic < min_cost_so_far:
min_cost_so_far, min_index = cost_heuristic, i
path_cost, path = agenda.pop(min_index)
else: # not use heuristic for find_fast_path
path_cost, path = agenda.pop(
min(range(len(agenda)), key=agenda.__getitem__))
# pulled_agenda += 1
terminal = path[-1]
if terminal not in expanded:
if terminal == end: # found the path, return list of tuples nodes locations: (lat, lon)
#print(pulled_agenda)
#return path
return [graph[node_id]['loc'] for node_id in path]
expanded.add(terminal)
for adj in graph[terminal]:
if adj not in expanded and adj in graph:
new_cost = great_circle_distance(graph[adj]['loc'],
graph[terminal]['loc'])
if not short: # fastest path, calculate terminal->adj cost = dist/speed = time travel
new_cost /= graph[terminal][adj] # speed
agenda.append((path_cost + new_cost, path + [adj]))
return None
def find_short_path(aux_structures, loc1, loc2):
"""
Return the shortest path between the two locations
Parameters:
aux_structures: the result of calling build_auxiliary_structures
loc1: tuple of 2 floats: (latitude, longitude), representing the start
location
loc2: tuple of 2 floats: (latitude, longitude), representing the end
location
Returns:
a list of (latitude, longitude) tuples representing the shortest path
(in terms of distance) from loc1 to loc2.
"""
Nodes, Edges, Dic = aux_structures
nodes1 = nearest(Dic, loc1)
nodes2 = nearest(Dic, loc2)
agenda = [[[Nodes[nodes1]], 0, great_circle_distance(loc1, loc2)]]
# agenda = [[[Nodes[nodes1]], 0]]
empty = set()
count = 0
while agenda:
# Remove the path with the lowest cost from the agenda.
path = agenda.pop(0)
count += 1
# If this path's terminal vertex is in the expanded set，
# ignore it completely and move on to the next path.
if path[0][-1] in empty:
continue
# If this path's terminal vertex satisfies the goal condition
# return that path. Otherwise, add its terminal vertex to the expanded set.
if path[0][-1] == Nodes[nodes2]:
print(count)
return path[0]
else:
empty.add(path[0][-1])
# For each of the children of that path's terminal vertex:
id = Dic[path[0][-1]]
for neighbor in Edges[id]:
# If it is in the expanded set, skip it
# Otherwise, add the associated path (and cost) to the agenda
if Nodes[neighbor[0]] not in empty:
p = []
for i in path[0]:
p.append(i)
p.append(Nodes[neighbor[0]])
dis = great_circle_distance(Nodes[id], Nodes[neighbor[0]])
h = great_circle_distance(Nodes[neighbor[0]], loc2)
agenda.append([p, dis + path[1], h + dis + path[1]])
# agenda.append([p, dis+path[1]])
agenda.sort(key=lambda x: x[-1])
return None
def find_short_path(aux_structures, loc1, loc2, n=0, oth_nodes=node_list):
"""
Return the shortest path between the two locations
Parameters:
aux_structures: the result of calling build_auxiliary_structures
loc1: tuple of 2 floats: (latitude, longitude), representing the start
location
loc2: tuple of 2 floats: (latitude, longitude), representing the end
location
Returns:
a list of (latitude, longitude) tuples representing the shortest path
(in terms of distance) from loc1 to loc2.
"""
the_nodes = oth_nodes.copy()
nodes_to_remove = set()
for s in the_nodes:
if not (s[0] in aux_structures.keys()):
nodes_to_remove.add(s)
the_nodes = the_nodes.difference(nodes_to_remove)
min_dist1 = -1
min_dist2 = -1
min1 = 0
min2 = 0
for r in the_nodes:
#parsing through every node to search for least distance to a node.
one_dist = great_circle_distance((r[1], r[2]), (loc1[0], loc1[1]))
two_dist = great_circle_distance((r[1], r[2]), (loc2[0], loc2[1]))
if (one_dist < min_dist1) or (min_dist1 == -1):
min_dist1 = one_dist
min1 = r[0]
if (two_dist < min_dist2) or (min_dist2 == -1):
min_dist2 = two_dist
min2 = r[0]
#find the shortest path between these nodes
if n != 0:
real_path = find_short_path_nodes(aux_structures, min1, min2, 1)
else:
real_path = find_short_path_nodes(aux_structures, min1, min2)
if real_path is None or len(real_path) == 1:
return None
#convert node ids to coordinates
for q in the_nodes:
if q[0] in real_path:
the_index = real_path.index(q[0])
new_tuple = (q[1], q[2])
real_path.remove(q[0])
real_path.insert(the_index, new_tuple)
return real_path
def nearest(Dic, loc1):
"""
Return the nearest node to loc1 in Dic
"""
node = None
loc = None
# Loop over i in Dic.keys()
for i in Dic.keys():
if loc == None or great_circle_distance(
loc, loc1) > great_circle_distance(i, loc1):
# update node and loc
node = Dic[i]
loc = i
return node
def find_start_end(loc1, loc2):
"""
Find start/end nodes near loc1/loc2, by iterating over a graph
of all valid nodes (those appeared on some ways); and find smallest great_circle_distance
"""
start = end = None # start node, end node
min_start_distance = min_end_distance = float('inf')
for node_id in graph:
node_loc = graph[node_id]['loc']
distance_start = great_circle_distance(loc1, node_loc)
distance_end = great_circle_distance(loc2, node_loc)
if distance_start < min_start_distance:
start, min_start_distance = node_id, distance_start
if distance_end < min_end_distance:
end, min_end_distance = node_id, distance_end
return start, end
def get_distance(node_db, node1, node2):
'''
Returns the distance in miles between two Node ids
'''
coord1 = node_db[node1]['coordinates']
coord2 = node_db[node2]['coordinates']
return great_circle_distance(coord1, coord2)
def cost_between(dist, speed, n1, n2, short=True):
d = great_circle_distance(dist[n1], dist[n2])
if short:
return d
else:
pair = sorted_tuple(n1, n2)
s = speed[pair]
return d / s
def nearest_node_to_loc(node_locs, loc):
best = float('inf')
node = None
for id_, nloc in node_locs.items():
dist = great_circle_distance(loc, nloc)
if dist < best:
best = dist
node = id_
return node
def find_fast_path(aux_structures, loc1, loc2):
"""
loc1 and loc2 are (lat, lon) tuples
"""
node_locs, adjacency, max_speed_limit = aux_structures
node1 = nearest_node_to_loc(node_locs, loc1)
node2 = nearest_node_to_loc(node_locs, loc2)
print('running search')
path = a_star(adjacency,
node1,
node2,
cost_func=lambda n1, n2: great_circle_distance(
node_locs[n1], node_locs[n2[0]]) / n2[1],
heuristic=lambda node: great_circle_distance(
node_locs[node], node_locs[node2]) / max_speed_limit)
return [node_locs[n] for n in path] if path is not None else None
def get_closest_node(aux_structures, loc):
locs = aux_structures['locations']
best = None
min_cost = None
for id, nloc in locs.items():
cost = great_circle_distance(nloc, loc)
if best is None or cost < min_cost:
best = id
min_cost = cost
return best
def calculate_distance(aux_structures, start, end):
"""
Returns the distance between two nodes.
start and end are node IDs.
"""
start_id_to_coordinate = map_node_id_to_coordinates(aux_structures, start)
end_id_to_coordinate = map_node_id_to_coordinates(aux_structures, end)
distance = great_circle_distance(start_id_to_coordinate,
end_id_to_coordinate)
return distance
def spherematch2(ra1, dec1, ra2, dec2, kdt, tolerance=1/3600., k=100): """ Uses a k-d tree to efficiently match two pairs of coordinates in spherical geometry, with a tolerance in degrees. """ coords1 = radec_to_coords(ra1, dec1) idx = kdt.query(coords1, k=k)[1].flatten() ds = great_circle_distance(ra1, dec1, ra2[idx], dec2[idx]) msk = ds < tolerance idx = idx[msk] ds = ds[msk] return idx, ds
def spherematch2(ra1, dec1, ra2, dec2, kdt, tolerance=1 / 3600., k=100):
"""
Uses a k-d tree to efficiently match two pairs of coordinates in spherical
geometry, with a tolerance in degrees.
"""
coords1 = radec_to_coords(ra1, dec1)
idx = kdt.query(coords1, k=k)[1].flatten()
ds = great_circle_distance(ra1, dec1, ra2[idx], dec2[idx])
msk = ds < tolerance
idx = idx[msk]
ds = ds[msk]
return idx, ds
def cull_bad_measurements(phot_groups_filepath):
"""
Eliminates bad measurements from the groups of measurements
for each source. Uses a SNR cutoff and a proximity cutoff,
such that only sources with SNR above min_snr and with centroids
less than max_dist from the input RA/Dec position will survive.
max_dist is in units of arcsec.
"""
work_dir = "/".join(phot_groups_filepath.split("/")[:-1])
meta = json.load(open(work_dir + "/metadata.json"))
min_snr = meta["min_snr"]
max_dist = meta["max_dist"]
# read in the photometry JSON files
ch = json.load(open(phot_groups_filepath))
# loop through the sources and look for measurements to cull
rejected = {}
badkeys = []
for key in ch:
good, bad = [], []
for obs in ch[key]:
ra_inp, dec_inp = obs[:2]
ra_cnt, dec_cnt = obs[2:4]
snr = obs[6] / obs[7]
d = great_circle_distance(ra_inp, dec_inp, ra_cnt, dec_cnt) * 3600
if snr > min_snr and d < max_dist:
good.append(obs)
else:
bad.append(obs)
if len(good) > 0:
ch[key] = good
else:
badkeys.append(key)
if len(bad) > 0:
rejected[key] = bad
# eliminate source altogether if all its measurements get culled
for key in badkeys:
ch.pop(key)
# write to disk
out_path = work_dir + "/phot_groups_culled.json"
with open(out_path, "w") as w:
json.dump(ch, w, indent=4 * " ")
print("created file: " + out_path)
out_path = work_dir + "/phot_groups_rejected.json"
with open(out_path, "w") as w:
json.dump(rejected, w, indent=4 * " ")
print("created file: " + out_path)
def find_nearest_nodes(valid_ids, loc1, loc2):
'''
Takes two locations. Finds the nearest id to each loc by computing the
minimum distance from loc to all our ids. Only stores the minimum
distance (to compare it) and the corresponding minimum id, which
it will actualy return. Does this for both loc1 and loc2.
'''
inputs = [loc1, loc2]
minimum = float('inf')
out = []
for value in inputs:
for ID in valid_ids:
location = valid_ids[ID]['location']
if great_circle_distance(value, location) < minimum:
minimum = great_circle_distance(value, location)
minimum_id = ID
out.append(minimum_id)
minimum = float(
'inf') #redeclares the minimum to correctly evaluate for loc2
return out[0], out[1]
def find_nearest_node(loc, aux_structures):
'''
Finds nearest node. Takes in location (lat, lon) and a dictionary of nodes_web (in aux structures)
'''
min_dist = float("inf")
node_coords = aux_structures[1] #node_id: coord
for node in node_coords.keys():
d = great_circle_distance((node_coords[node][0], node_coords[node][1]),
loc)
if d < min_dist: # finds min distance from node to loc
min_dist = d
n1 = node
return n1
def min_cost_heuristic(agenda, dist, goal):
min_cost = None
min_node = None
for node in agenda:
cost = agenda[node] + great_circle_distance(dist[goal], dist[node])
if min_cost is None:
min_cost = cost
min_node = node
else:
if cost < min_cost:
min_cost = cost
min_node = node
return min_node
def get_dist_cost(data, start_node_id, end_node_id):
"""
Calculates the cost of the direct path (which is assume to exist) between
specified start and end nodes based on the distance between them.
Parameters:
data: The auxiliary data structure (a dictionary) that stores information
about nodes and the ways that connect them
start_node_id: The integer id of the start node in data
end_node_id: The integer id of the end node in data
"""
p1 = get_coords(data, start_node_id)
p2 = get_coords(data, end_node_id)
return great_circle_distance(p1, p2)
def find_short_path_nodes(aux_structures, node1, node2):
"""
Return the shortest path between the two nodes
Parameters:
aux_structures: the result of calling build_auxiliary_structures
node1: node representing the start location
node2: node representing the end location
Returns:
a list of node IDs representing the shortest path (in terms of
distance) from node1 to node2
"""
considered_paths = [([node1], 0, 0)]
previous_nodes = set()
while len(considered_paths) != 0:
considered_paths.sort(key=lambda tup: tup[1] + tup[2])
shortest_value = considered_paths.pop(0)
if shortest_value[0][-1] in previous_nodes:
continue
if shortest_value[0][-1] == node2:
return shortest_value[0]
else:
previous_nodes.add(shortest_value[0][-1])
for node in aux_structures[0].get(shortest_value[0][-1], {}):
loc1 = aux_structures[1][shortest_value[0][-1]]
loc2 = aux_structures[1][node]
cost = great_circle_distance(loc1, loc2)
if node in previous_nodes:
continue
else:
considered_paths.append(
(shortest_value[0] + [node], shortest_value[1] + cost,
great_circle_distance(loc2,
aux_structures[1][node2])))
return None
def get_nearest_node(nodes_db, lat, lon):
"""
Returns the nearest node that is in a relevant way
Parameters:
nodes_db: A node database which should be the first element when calling build_auxiliary_database on a database
lat: a latitude coordinate
lon: a longitude coordinate
Returns:
a node's ID that is the closest to the specified coordinate
"""
key_min = min(
nodes_db,
key=lambda x: great_circle_distance(nodes_db[x]['coordinates'],
(lat, lon)))
return key_min
def get_closest_node(data, loc):
"""
Calculates the closest node in the given dataset to a specified query location
Parameters:
data: The auxiliary data structure (a dictionary) that stores information
about nodes and the ways that connect them
loc: The query location, given in terms of a tuple of two floats (latitude, longitude)
"""
min_dist = None
closest = None
for i in data:
# Standard min-value search loop
dist = great_circle_distance(get_coords(data, i), loc)
if closest is None or dist < min_dist:
closest = i
min_dist = dist
return closest
def find_nearest_node_id(aux_structures, loc):
"""
Returns the node ID for the nearest node next to loc (chosen from all the valid nodes).
loc: tuple of 2 floats: (latitude, longitude)
"""
nodes, ways, max_speed_dic = aux_structures
distances = {}
# Map node IDs to distances
for n in nodes:
lat, lon = nodes[n]['lat'], nodes[n]['lon']
distances[n] = great_circle_distance(loc, (lat, lon))
# Find min distance
min_distance = min(distances.values())
# Get node ID with min distance
for node_id, distance in distances.items():
if distance == min_distance:
return node_id
def get_nearest_node_id(aux_structures, nodes, n1):
"""
Returns the node ID for the nearest node next to n1 from a set of nodes.
n1: node ID.
nodes: set containing node IDs.
"""
id_coordinate_map = map_node_id_to_coordinates(aux_structures)
distances = {}
# Map node IDs to distances
for n in nodes:
distances[n] = great_circle_distance(loc, id_coordinate_map[n])
# Find min distance
min_distance = min(distances.values())
# Get node ID with min distance
for node_id, distance in distances.items():
if distance == min_distance:
return node_id
def find_nearest_loc(aux, loc1, loc2):
dist = aux[1]
# find n1 and n2 (nearest nodes to the given loc1 and loc2)
d1 = None
n1 = None
d2 = None
n2 = None
for n in dist:
if d1 is None:
d1 = great_circle_distance(dist[n], loc1)
n1 = n
d2 = great_circle_distance(dist[n], loc2)
n2 = n
else:
if great_circle_distance((dist[n]), loc1) < d1:
d1 = great_circle_distance(dist[n], loc1)
n1 = n
if great_circle_distance((dist[n]), loc2) < d2:
d2 = great_circle_distance(dist[n], loc2)
n2 = n
return n1, n2
def match_sdss(cat_path):
for catfile in find_files(cat_path, "*merged.txt"):
# read pipeline catalog
print("\nreading catalog: {}".format(catfile))
cat = pd.read_table(catfile, sep=' ')
# retrieve SDSS data from ViZieR if not already downloaded
ch = catfile.split('/')[-1].split('_')[1]
outpath = catfile.replace('{}_merged.txt'.format(ch), 'sdss.vot')
if not os.path.isfile(outpath):
cntr_ra = np.median(cat.ra)
cntr_dec = np.median(cat.dec)
# get source from one corner of the mosaic to calculate radius
c1 = (cat.ra.min(), cat.dec[cat.ra == cat.ra.min()].values[0])
# make radius 10% bigger just to be on safe side
radius = great_circle_distance(cntr_ra, cntr_dec, *c1) * 1.1
url = get_url(cntr_ra, cntr_dec, radius)
print("retrieving URL: {}".format(url))
handler = urllib2.urlopen(url)
raw = handler.read()
with open(outpath, 'wb') as f:
f.write(raw)
print("created file: {}".format(outpath))
# parse VOTable
print("reading VOTable: {}".format(outpath))
table = parse_single_table(outpath)
# if this is one of the southern hemisphere regions, delete and continue
if table.array.size == 0:
os.remove(outpath)
print("outside of SDSS coverage")
continue
# make sure no missing data
for name in table.array.dtype.names:
assert table.array[name].mask.sum() == 0
# get unmasked array
sdss = table.array.data
# make sure sky coverage is big enough
assert sdss['RAJ2000'].min() < cat.ra.min()
assert sdss['RAJ2000'].max() > cat.ra.max()
assert sdss['DEJ2000'].min() < cat.dec.min()
assert sdss['DEJ2000'].max() > cat.dec.max()
# match to catalog
assert cat.shape[0] < sdss.shape[0]
tol = 2 / 3600.
idx1, idx2, ds = spherematch(cat.ra,
cat.dec,
sdss['RAJ2000'],
sdss['DEJ2000'],
tolerance=tol)
print("matched {} out of {} sources with {} arcsec tolerance".format(
ds.size, cat.shape[0], tol * 3600))
# create vector of star/galaxy class (0=missing, 3=galaxy, 6=star)
cl = np.zeros(cat.shape[0]).astype('int')
cl[idx1] = sdss['cl'][idx2]
# add the column to the dataset
cat['cl'] = cl
# write to new file
outpath = catfile.replace('merged.txt', 'merged+sdss.txt')
# fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']*2
# hdr = ' '.join(names)+' cl'
# np.savetxt(outpath, df.to_records(index=False), fmt=fmt, header=hdr)
cat.to_csv(outpath, index=False, sep=' ', float_format='%.8f')
print("created file: {}".format(outpath))
def assert_distance(self, p1, p2, expected_distance):
distance = great_circle_distance(p1, p2)
self.assertAlmostEqual(expected_distance, distance)
def match_wise(cat_path, sdss=True):
if sdss:
search_pattern = "*merged+sdss.txt"
else:
search_pattern = "*merged.txt"
for catfile in find_files(cat_path, search_pattern):
# read pipeline catalog
print("\nreading catalog: {}".format(catfile))
cat = pd.read_table(catfile, sep=' ')
# retrieve WISE data from ViZieR if not already downloaded
ch = catfile.split('/')[-1].split('_')[1]
if sdss:
outpath = catfile.replace('{}_merged+sdss.txt'.format(ch),
'wise.vot')
else:
outpath = catfile.replace('{}_merged.txt'.format(ch), 'wise.vot')
if not os.path.isfile(outpath):
cntr_ra = np.median(cat.ra)
cntr_dec = np.median(cat.dec)
# get source from one corner of the mosaic to calculate radius
c1 = (cat.ra.min(), cat.dec[cat.ra == cat.ra.min()].values[0])
# make radius 10% bigger just to be on safe side
radius = great_circle_distance(cntr_ra, cntr_dec, *c1) * 1.1
url = get_url(cntr_ra, cntr_dec, radius)
print("retrieving URL: {}".format(url))
handler = urllib2.urlopen(url)
raw = handler.read()
with open(outpath, 'wb') as f:
f.write(raw)
print("created file: {}".format(outpath))
# parse VOTable
print("reading VOTable: {}".format(outpath))
table = parse_single_table(outpath)
# if this is one of the southern hemisphere regions, delete and continue
if table.array.size == 0:
os.remove(outpath)
print("no WISE coverage")
continue
# get unmasked array
wise = table.array.data
# make sure sky coverage is big enough
assert wise['RAJ2000'].min() < cat.ra.min()
assert wise['RAJ2000'].max() > cat.ra.max()
assert wise['DEJ2000'].min() < cat.dec.min()
assert wise['DEJ2000'].max() > cat.dec.max()
# match to catalog
tol = 2 / 3600.
if cat.shape[0] < wise.shape[0]:
idx1, idx2, ds = spherematch(cat.ra,
cat.dec,
wise['RAJ2000'],
wise['DEJ2000'],
tolerance=tol)
else:
idx2, idx1, ds = spherematch(wise['RAJ2000'],
wise['DEJ2000'],
cat.ra,
cat.dec,
tolerance=tol)
print("matched {} out of {} sources with {} arcsec tolerance".format(
ds.size, cat.shape[0], tol * 3600))
# add WISE to the catalog
if ch == '1':
cat['W1mag'] = np.repeat(np.nan, cat.shape[0])
cat['e_W1mag'] = np.repeat(np.nan, cat.shape[0])
cat['W1mag'][idx1] = wise['W1mag'][idx2]
cat['e_W1mag'][idx1] = wise['e_W1mag'][idx2]
elif ch == '2':
cat['W2mag'] = np.repeat(np.nan, cat.shape[0])
cat['e_W2mag'] = np.repeat(np.nan, cat.shape[0])
cat['W2mag'][idx1] = wise['W2mag'][idx2]
cat['e_W2mag'][idx1] = wise['e_W2mag'][idx2]
else:
print("unexpected error adding WISE data")
# write to new file
outpath = catfile.replace('.txt', '+wise.csv')
# fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']*2
# hdr = ' '.join(names)+' cl'
# np.savetxt(outpath, df.to_records(index=False), fmt=fmt, header=hdr)
cat.to_csv(outpath, index=False, float_format='%.8f')
print("created file: {}".format(outpath))
def match_wise(cat_path, sdss=True):
if sdss:
search_pattern = "*merged+sdss.txt"
else:
search_pattern = "*merged.txt"
for catfile in find_files(cat_path, search_pattern):
# read pipeline catalog
print("\nreading catalog: {}".format(catfile))
cat = pd.read_table(catfile, sep=' ')
# retrieve WISE data from ViZieR if not already downloaded
ch = catfile.split('/')[-1].split('_')[1]
if sdss:
outpath = catfile.replace('{}_merged+sdss.txt'.format(ch), 'wise.vot')
else:
outpath = catfile.replace('{}_merged.txt'.format(ch), 'wise.vot')
if not os.path.isfile(outpath):
cntr_ra = np.median(cat.ra)
cntr_dec = np.median(cat.dec)
# get source from one corner of the mosaic to calculate radius
c1 = (cat.ra.min(), cat.dec[cat.ra==cat.ra.min()].values[0])
# make radius 10% bigger just to be on safe side
radius = great_circle_distance(cntr_ra, cntr_dec, *c1) * 1.1
url = get_url(cntr_ra, cntr_dec, radius)
print("retrieving URL: {}".format(url))
handler = urllib2.urlopen(url)
raw = handler.read()
with open(outpath,'wb') as f:
f.write(raw)
print("created file: {}".format(outpath))
# parse VOTable
print("reading VOTable: {}".format(outpath))
table = parse_single_table(outpath)
# if this is one of the southern hemisphere regions, delete and continue
if table.array.size == 0:
os.remove(outpath)
print("no WISE coverage")
continue
# get unmasked array
wise = table.array.data
# make sure sky coverage is big enough
assert wise['RAJ2000'].min() < cat.ra.min()
assert wise['RAJ2000'].max() > cat.ra.max()
assert wise['DEJ2000'].min() < cat.dec.min()
assert wise['DEJ2000'].max() > cat.dec.max()
# match to catalog
tol = 2/3600.
if cat.shape[0] < wise.shape[0]:
idx1, idx2, ds = spherematch(cat.ra, cat.dec,
wise['RAJ2000'], wise['DEJ2000'], tolerance = tol)
else:
idx2, idx1, ds = spherematch(wise['RAJ2000'], wise['DEJ2000'],
cat.ra, cat.dec, tolerance = tol)
print("matched {} out of {} sources with {} arcsec tolerance".format(ds.size,
cat.shape[0], tol*3600))
# add WISE to the catalog
if ch == '1':
cat['W1mag'] = np.repeat(np.nan, cat.shape[0])
cat['e_W1mag'] = np.repeat(np.nan, cat.shape[0])
cat['W1mag'][idx1] = wise['W1mag'][idx2]
cat['e_W1mag'][idx1] = wise['e_W1mag'][idx2]
elif ch == '2':
cat['W2mag'] = np.repeat(np.nan, cat.shape[0])
cat['e_W2mag'] = np.repeat(np.nan, cat.shape[0])
cat['W2mag'][idx1] = wise['W2mag'][idx2]
cat['e_W2mag'][idx1] = wise['e_W2mag'][idx2]
else:
print("unexpected error adding WISE data")
# write to new file
outpath = catfile.replace('.txt', '+wise.csv')
# fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']*2
# hdr = ' '.join(names)+' cl'
# np.savetxt(outpath, df.to_records(index=False), fmt=fmt, header=hdr)
cat.to_csv(outpath, index=False, float_format='%.8f')
print("created file: {}".format(outpath))
def build_auxiliary_structures(nodes_filename, ways_filename):
"""
Create any auxiliary structures you are interested in, by reading the data
from the given filenames (using read_osm_data)
"""
#The datatype I will be a dict with keys of eligible node ids
#with values containing a set of tuples of adjacent eligible nodes:
#{node: {(id,lat,lon,distance,speed_limit)}}
answer_dict = {}
big_nodes = {}
list_of_nodes = set()
for z in read_osm_data(ways_filename):
if 'highway' in z['tags'].keys():
if z['tags']['highway'] in ALLOWED_HIGHWAY_TYPES:
for f in z['nodes']:
list_of_nodes.add(f)
for y in read_osm_data(nodes_filename):
if y['id'] in list_of_nodes:
big_nodes.update({y['id']: y})
big_tuple = (y['id'], y['lat'], y['lon'])
node_list.add(big_tuple)
list_of_nodes.clear()
for x in read_osm_data(ways_filename):
if 'highway' in x['tags'].keys():
if x['tags']['highway'] in ALLOWED_HIGHWAY_TYPES:
for w in range(len(x['nodes'])):
#find speed limit:
if 'maxspeed_mph' in x['tags'].keys():
speed = x['tags']['maxspeed_mph']
else:
speed = DEFAULT_SPEED_LIMIT_MPH[x['tags']['highway']]
#adds a new key if id is not in answer_dict
if x['nodes'][w] not in answer_dict.keys():
answer_dict.update({x['nodes'][w]: set()})
if w != len(x['nodes']) - 1:
#calculate distance between x['nodes'][w] and x['nodes'][w+1]
lat = big_nodes[x['nodes'][w + 1]]['lat']
lon = big_nodes[x['nodes'][w + 1]]['lon']
oth_lat = big_nodes[x['nodes'][w]]['lat']
oth_lon = big_nodes[x['nodes'][w]]['lon']
dist = great_circle_distance((lat, lon),
(oth_lat, oth_lon))
answer_dict[x['nodes'][w]].add(
(x['nodes'][w + 1], lat, lon, dist, speed))
'''
#to avoid duplicate reference nodes.
length=None
sonic=None
for f in answer_dict[x['nodes'][w]]:
if f[0]==x['nodes'][w+1]:
length=f[3]
sonic=f[4]
if length==None:
answer_dict[x['nodes'][w]].add((x['nodes'][w+1],lat,lon,dist,speed))
elif (length/sonic)>(dist/speed):
answer_dict[x['nodes'][w]].remove((x['nodes'][w+1],lat,lon,length,sonic))
answer_dict[x['nodes'][w]].add((x['nodes'][w+1],lat,lon,dist,speed))
'''
if 'oneway' in x['tags'].keys():
if x['tags']['oneway'] != 'yes':
if w != 0:
#calculate distance between x['nodes'][w] and x['nodes'][w-1]
lat = big_nodes[x['nodes'][w - 1]]['lat']
lon = big_nodes[x['nodes'][w - 1]]['lon']
oth_lat = big_nodes[x['nodes'][w]]['lat']
oth_lon = big_nodes[x['nodes'][w]]['lon']
dist = great_circle_distance(
(lat, lon), (oth_lat, oth_lon))
answer_dict[x['nodes'][w]].add(
(x['nodes'][w - 1], lat, lon, dist, speed))
'''
#to avoid duplicate reference nodes.
length1=None
sonic1=None
for f in answer_dict[x['nodes'][w]]:
if f[0]==x['nodes'][w-1]:
length1=f[3]
sonic1=f[4]
if length1==None:
answer_dict[x['nodes'][w]].add((x['nodes'][w-1],lat,lon,dist,speed))
elif (length1/sonic1)>(dist/speed):
answer_dict[x['nodes'][w]].remove((x['nodes'][w-1],lat,lon,length1,sonic1))
answer_dict[x['nodes'][w]].add((x['nodes'][w-1],lat,lon,dist,speed))
'''
else:
if w != 0:
#calculate distance between x['nodes'][w] and x['nodes'][w-1]
lat = big_nodes[x['nodes'][w - 1]]['lat']
lon = big_nodes[x['nodes'][w - 1]]['lon']
oth_lat = big_nodes[x['nodes'][w]]['lat']
oth_lon = big_nodes[x['nodes'][w]]['lon']
dist = great_circle_distance((lat, lon),
(oth_lat, oth_lon))
answer_dict[x['nodes'][w]].add(
(x['nodes'][w - 1], lat, lon, dist, speed))
'''
#to avoid duplicate reference nodes.
length2=None
sonic2=None
for f in answer_dict[x['nodes'][w]]:
if f[0]==x['nodes'][w-1]:
length2=f[3]
sonic2=f[4]
if length2==None:
answer_dict[x['nodes'][w]].add((x['nodes'][w-1],lat,lon,dist,speed))
elif (length2/sonic2)>(dist/speed):
answer_dict[x['nodes'][w]].remove((x['nodes'][w-1],lat,lon,length2,sonic2))
answer_dict[x['nodes'][w]].add((x['nodes'][w-1],lat,lon,dist,speed))
'''
big_nodes.clear()
#removing any empty keys
actual_dict = {}
print(node_list)
for f in answer_dict.keys():
if answer_dict[f] != set():
actual_dict.update({f: answer_dict[f]})
answer_dict.clear()
return actual_dict
def gcd_heuristic(data, node1, node2):
return great_circle_distance(get_coords(data, node1),
get_coords(data, node2))
def match_sdss(cat_path):
for catfile in find_files(cat_path, "*merged.txt"):
# read pipeline catalog
print("\nreading catalog: {}".format(catfile))
cat = pd.read_table(catfile, sep=' ')
# retrieve SDSS data from ViZieR if not already downloaded
ch = catfile.split('/')[-1].split('_')[1]
outpath = catfile.replace('{}_merged.txt'.format(ch), 'sdss.vot')
if not os.path.isfile(outpath):
cntr_ra = np.median(cat.ra)
cntr_dec = np.median(cat.dec)
# get source from one corner of the mosaic to calculate radius
c1 = (cat.ra.min(), cat.dec[cat.ra==cat.ra.min()].values[0])
# make radius 10% bigger just to be on safe side
radius = great_circle_distance(cntr_ra, cntr_dec, *c1) * 1.1
url = get_url(cntr_ra, cntr_dec, radius)
print("retrieving URL: {}".format(url))
handler = urllib2.urlopen(url)
raw = handler.read()
with open(outpath,'wb') as f:
f.write(raw)
print("created file: {}".format(outpath))
# parse VOTable
print("reading VOTable: {}".format(outpath))
table = parse_single_table(outpath)
# if this is one of the southern hemisphere regions, delete and continue
if table.array.size == 0:
os.remove(outpath)
print("outside of SDSS coverage")
continue
# make sure no missing data
for name in table.array.dtype.names:
assert table.array[name].mask.sum() == 0
# get unmasked array
sdss = table.array.data
# make sure sky coverage is big enough
assert sdss['RAJ2000'].min() < cat.ra.min()
assert sdss['RAJ2000'].max() > cat.ra.max()
assert sdss['DEJ2000'].min() < cat.dec.min()
assert sdss['DEJ2000'].max() > cat.dec.max()
# match to catalog
assert cat.shape[0] < sdss.shape[0]
tol = 2/3600.
idx1, idx2, ds = spherematch(cat.ra, cat.dec,
sdss['RAJ2000'], sdss['DEJ2000'], tolerance = tol)
print("matched {} out of {} sources with {} arcsec tolerance".format(ds.size,
cat.shape[0], tol*3600))
# create vector of star/galaxy class (0=missing, 3=galaxy, 6=star)
cl = np.zeros(cat.shape[0]).astype('int')
cl[idx1] = sdss['cl'][idx2]
# add the column to the dataset
cat['cl'] = cl
# write to new file
outpath = catfile.replace('merged.txt', 'merged+sdss.txt')
# fmt = ['%i']+['%0.8f']*2+['%.4e']*2+['%i']*2
# hdr = ' '.join(names)+' cl'
# np.savetxt(outpath, df.to_records(index=False), fmt=fmt, header=hdr)
cat.to_csv(outpath, index=False, sep=' ', float_format='%.8f')
print("created file: {}".format(outpath))
#------------------------------------------------------------------------------
angles = np.linspace(0, 2 * np.pi, image_size[0])
height = np.linspace(0, 1., image_size[1])
angle_grid, height_grid = np.meshgrid(angles, height)
#------------------------------------------------------------------------------
# Convert cylinder coordinates to spherical coordinates
#------------------------------------------------------------------------------
spherical_points = util.cylindric_to_spheric(height_grid, angle_grid)
#------------------------------------------------------------------------------
# Compute distances to each randomly generated point on the sphere
# Keep the minimal distance for each pixel
#------------------------------------------------------------------------------
dots = util.iter_dots_on_sphere(max_dot_n, dot_min_distance)
min_distances = util.great_circle_distance(spherical_points, next(dots))
for dot in dots:
distances = util.great_circle_distance(spherical_points, dot)
np.minimum(distances, min_distances, out=min_distances)
#------------------------------------------------------------------------------
# Use the float array to create an array of uint8, containing either black if
# the minimal distance for a given pixel to any of the points is below
# the dot_r or white otherwise
#------------------------------------------------------------------------------
black = np.zeros(image_size, dtype=np.uint8)
white = np.ones(image_size, dtype=np.uint8) * 255
spherical_projected_texture = np.where(min_distances.T < dot_r, black, white)
#------------------------------------------------------------------------------
# Save the array to an image
