def data_from_coordinate_pair(save_dir, x, y):
# load the global tree, find the zone_id to which this star belongs
global_tree_file = save_dir + "/global.json"
global_tree = QuadTreeNode.from_file(global_tree_file, leafClass=IDLeaf)
leaf = global_tree.find_leaf(x, y)
zone_id = leaf.node_id
# load the zone, extract the star's data
zone_name = apass.name_zone(zone_id)
# load the zone's tree and data from disk and get the leaves
zone_json = save_dir + apass.name_zone_json_file(zone_id)
zone_tree = QuadTreeNode.from_file(zone_json, leafClass=RectLeaf)
zone.load_zone_data(zone_tree, save_dir)
# find the leaf containing the point of interest
leaf = zone_tree.find_leaf(x, y)
# export the data
data = None
for container in leaf.containers:
if container.contains(x, y):
data = fred.to_fredbin(container.data)
return data
def load_zone(save_dir, zone_id):
"""Loads the tree, data, and border info file for the specified zone.
Returns this data as a dictionary keyed as follows:
* json_filename
* container_filename
* border_info_filename
* border_info
* tree
* loaded
* lock (a FileLock instance)"""
#print(" Loading zone %i" % (zone_id))
zone_dict = dict()
zone_tree = None
zone_border_info = None
load_succeeded = True
lock = None
zone_json_file = save_dir + apass.name_zone_json_file(zone_id)
zone_container_file = save_dir + apass.name_zone_container_file(zone_id)
zone_border_info_file = save_dir + apass.name_zone_border_file(zone_id)
#print zone_json_file
#print zone_container_file
#print zone_border_info_file
lock = FileLock(zone_container_file)
try:
lock.acquire()
except FileLockException:
print(
"ERROR: Failed to get lock on zone %i, check for stale lockfiles!"
% (zone_id))
quit()
# verify the necessary files exist, if not bail
if not os.path.isfile(zone_json_file):
# print(" WARNING: JSON file missing for zone %i" % (zone_id))
load_succeeded = False
if not os.path.isfile(zone_border_info_file):
# print(" WARNING: Border Info file missing for zone %i" % (zone_id))
load_succeeded = False
if not os.path.isfile(zone_container_file):
# print(" WARNING: Container data file missing for zone %i" % (zone_id))
load_succeeded = False
if load_succeeded:
zone_border_info = load_border_info(zone_border_info_file)
zone_tree = QuadTreeNode.from_file(zone_json_file, leafClass=RectLeaf)
load_zone_data(zone_tree, save_dir)
zone_dict['json_filename'] = zone_json_file
zone_dict['container_filename'] = zone_container_file
zone_dict['border_info_filename'] = zone_border_info_file
zone_dict['tree'] = zone_tree
zone_dict['border_info'] = zone_border_info
zone_dict['loaded'] = load_succeeded
zone_dict['lock'] = lock
return zone_dict
def apass_zone_to_dat(save_dir, filter_config, zone_container_filename):
"""Process all of the rectangles found in the zone.
With APASS data, we average on a per-container basis."""
averages = []
zone_id = apass.zone_from_name(zone_container_filename)
zone_name = apass.name_zone(zone_id)
print "Processing zone " + zone_name
# load the zone's tree and data from disk and get the leaves
zone_json = save_dir + apass.name_zone_json_file(zone_id)
zone_tree = QuadTreeNode.from_file(zone_json, leafClass=RectLeaf)
zone.load_zone_data(zone_tree, save_dir)
leaves = zone_tree.get_leaves()
# average the data in each container.
for leaf in leaves:
for container in leaf.containers:
# average the data
c_ave = average_container(container, filter_config)
averages.append(c_ave)
# write out the average information
dat_filename = save_dir + "/" + zone_name + ".dat"
averages = dat.dicts_to_ndarray(averages, dat_type="apass")
dat.write_dat(dat_filename, averages, dat_type="apass")
def zone_to_rects(save_dir, filename):
"""Processes and APASS zone file into overlapping rectangles"""
zone_id = apass.zone_from_name(filename)
global tree_file
# read in the (binary) data file
data = read_fredbin(filename)
print "Processing '%s' which has %i data points " % (filename, data.size)
# find the bounds of this zone using the first data point in the file
global_tree = QuadTreeNode.from_file(tree_file, leafClass=IDLeaf)
datum = data[0]
ra, dec = apass.get_coords(datum)
zone_node = global_tree.find_leaf(ra, dec)
zone_bounds = zone_node.rect
# build a tree for the zone
zone_tree = QuadTreeNode(zone_bounds, 0, parent=None)
zone_tree.split_until(apass.zone_depth, leafClass=RectLeaf)
# insert the data into the tree, building up containers (rectangles) in the
# process
for datum in np.nditer(data):
datum = datum.copy()
ra, dec = apass.get_coords(datum)
try:
zone_tree.insert(ra, dec, datum)
except RuntimeError:
print("ERROR: Potential data corruption in " + filename)
print(
"ERROR: Check file, remove the zone directory, and re-run this program"
)
return
# prepare the save the data.
# the zone file's name
filename_no_ext = os.path.splitext(filename)[0]
# now number the (leaf) nodes
number_containers(zone_tree, zone_id=zone_id)
#plot_rects(zone_tree) # plot the nodes before the merge
zone_border_info = merge_containers_on_borders(zone_tree)
#plot_rects(zone_tree) # plot the nodes after the merge
# write out the containers that were on the border
filename = save_dir + '/' + apass.name_zone_border_file(zone_id)
save_border_info(filename, zone_border_info)
zone.save_zone_data(zone_tree, save_dir)
# save the zone -> container mapping
filename = save_dir + '/' + apass.name_zone_json_file(zone_id)
QuadTreeNode.to_file(zone_tree, filename)
def sro_zone_to_dat(save_dir, filter_config, zone_container_filename):
"""Processes all of the rectangles found in zone. Zone should be a valid subdirectory
of save_dir"""
zone_id = apass.zone_from_name(zone_container_filename)
zone_name = apass.name_zone(zone_id)
print "Processing zone " + zone_name
# create a graph data structure for this zone
G = nx.Graph()
# load the zone's tree and data from disk and get the leaves
zone_json = save_dir + apass.name_zone_json_file(zone_id)
zone_tree = QuadTreeNode.from_file(zone_json, leafClass=RectLeaf)
zone.load_zone_data(zone_tree, save_dir)
leaves = zone_tree.get_leaves()
# average the data and populate the graph with shared container information
line_number = 0
averages = []
for leaf in leaves:
for container in leaf.containers:
# average the data
c_aves = average_by_field(container, filter_config)
averages.extend(c_aves)
# populate overlapping line information
line_numbers = []
field_ids = []
for c_ave in c_aves:
line_numbers.append(line_number)
field_ids.append(c_ave['field_id'])
line_number += 1
# if there are more than one field ID present, populate information
# in the graph
if len(field_ids) > 1:
# generate all possible combinations of the line numbers and field IDs
# these express the edges in the graph
line_pairs = list(itertools.combinations(line_numbers, 2))
field_pairs = list(itertools.combinations(field_ids, 2))
for line_pair, field_pair in zip(line_pairs, field_pairs):
src_line = line_pair[0]
dst_line = line_pair[1]
src_field = field_pair[0]
dst_field = field_pair[1]
try:
edge = G[src_field][dst_field]
except:
G.add_edge(src_field, dst_field, line_ids=[], weight=0)
edge = G[src_field][dst_field]
edge['line_ids'].append((src_line, dst_line))
edge['weight'] += 1
# write out the average information
dat_filename = save_dir + "/" + zone_name + ".dat"
averages = dat.dicts_to_ndarray(averages, dat_type="sro")
dat.write_dat(dat_filename, averages, dat_type="sro")
# save the graph to a pickle file
graph_filename = save_dir + "/" + zone_name + ".p"
nx.write_gpickle(G, graph_filename)
def main():
"""Plots all data in an APASS zone and optionally displays container boundaries."""
parser = argparse.ArgumentParser(
description='Plots a zone file and its data')
parser.add_argument('input',
nargs='+',
help="Zone JSON files to be plotted")
parser.add_argument('--show-containers',
default=False,
action="store_true",
help="Plot borders for containers")
args = parser.parse_args()
save_dir = os.path.dirname(os.path.realpath(args.input[0])) + "/"
inputs = args.input
for filename in inputs:
zone_id = apass.zone_from_name(filename)
zone_name = apass.name_zone(zone_id)
print "Plotting zone " + zone_name
# load the original zone data. Note, we don't restore it to the tree
zone_data_file = save_dir + apass.name_zone_file(zone_id)
zone_data = fred.read_fredbin(zone_data_file)
print("Zone file has " + str(zone_data.size) + " entries")
# load the containerized zone data
zone_container_file = save_dir + apass.name_zone_container_file(
zone_id)
zone_container_data = fred.read_fredbin(zone_container_file)
print("Zone container file has " + str(zone_container_data.size) +
" entries")
# load the zone's tree
zone_json = save_dir + apass.name_zone_json_file(zone_id)
zone_tree = QuadTreeNode.from_file(zone_json, leafClass=RectLeaf)
leaves = zone_tree.get_leaves()
total_containers = 0
for leaf in leaves:
total_containers += len(leaf.containers)
print("Zone contains a total of " + str(total_containers) +
" containers")
fig, axes = plt.subplots(1)
# plot the zone container data
for leaf in leaves:
rect = leaf.rect
x = rect.x_min
y = rect.y_min
dx = rect.x_max - x
dy = rect.y_max - y
axes.add_patch(patches.Rectangle((x, y), dx, dy, fill=False))
if args.show_containers:
plot_containers(leaf, axes)
# plot the data
dec = zone_data['dec']
ra = zone_data['ra']
plt.scatter(ra, dec)
dec = zone_container_data['dec']
ra = zone_container_data['ra']
plt.scatter(ra, dec, color="green")
plt.show()