def summarize_fredbin(filename):
"""Reads in a fredbin file, prints out some basic statistics"""
print("Processing %s" % (filename))
output = init_summary_dict(filename)
data = None
try:
data = fred.read_fredbin(filename)
except:
pass
# bipass empty files
if data is None:
return output
# populate the dictionary with useful information
data = np.sort(data, order='night_name')
output['entries'] = len(data)
# find the unique named nights in the data
night_names = list(set(data['night_name']))
for night_name in night_names:
# find the upper and lower bounds for the night
l = np.searchsorted(data['night_name'], night_name, side='left')
r = np.searchsorted(data['night_name'], night_name, side='right')
output['nights'][night_name] = r - l
return output
def load_zone_data(tree, save_dir):
"""Restores zone data from the specified save_dir to the tree."""
# build a nested dictionary that gives us direct access to the
# underlying containers. Indexing shall be
# container = nodes[node_id][container_id]
node_dict = dict()
leaves = tree.get_leaves() # these will be of type RectLeaf
for leaf in leaves:
node_id = int(leaf.node_id)
container_dict = dict()
for container in leaf.containers:
container_id = int(container.container_id)
container_dict[container_id] = container
node_dict[node_id] = container_dict
# load the data
zone_id = leaves[0].zone_id
filename = save_dir + '/' + apass.name_zone_container_file(zone_id)
data = fred.read_fredbin(filename)
# insert the data *directly* into the container, bipassing normal
# restoration methods.
for i in range(0, len(data)):
datum = data[i]
node_id = int(datum['node_id'])
container_id = int(datum['container_id'])
node_dict[node_id][container_id].append_data(datum)
def purge_nights_from_file(night_names, filename):
results = []
if(not os.path.isfile(filename)):
return
print("Processing %s" % (filename))
# At least one common night exists. Load the data.
data = fred.read_fredbin(filename)
if data is None:
return results
# first determine if this night has been stored in this data file
data = np.sort(data, order='night_name')
for night_name in night_names:
# find the upper and lower bounds for the night
l = np.searchsorted(data['night_name'], night_name, side='left')
r = np.searchsorted(data['night_name'], night_name, side='right')
# delete the entries
data = np.delete(data, slice(l, r), axis=0)
# write the remaining data to disk
with open(filename, 'w') as outfile:
fred.write_fredbin(outfile, data)
return results
def data_from_unique_id(save_dir, zone_id, node_id, container_id):
zone_filename = save_dir + apass.name_zone(zone_id) + "-container.fredbin"
zone_data = fred.read_fredbin(zone_filename)
indices = np.where((zone_data['node_id'] == node_id)
& (zone_data['container_id'] == container_id))
data = zone_data[indices]
return data
def zone_to_rects(save_dir, filename):
"""Processes and APASS zone file into overlapping rectangles"""
zone_id = apass.zone_from_name(filename)
global tree_file
# read in the (binary) data file
data = read_fredbin(filename)
print "Processing '%s' which has %i data points " % (filename, data.size)
# find the bounds of this zone using the first data point in the file
global_tree = QuadTreeNode.from_file(tree_file, leafClass=IDLeaf)
datum = data[0]
ra, dec = apass.get_coords(datum)
zone_node = global_tree.find_leaf(ra, dec)
zone_bounds = zone_node.rect
# build a tree for the zone
zone_tree = QuadTreeNode(zone_bounds, 0, parent=None)
zone_tree.split_until(apass.zone_depth, leafClass=RectLeaf)
# insert the data into the tree, building up containers (rectangles) in the
# process
for datum in np.nditer(data):
datum = datum.copy()
ra, dec = apass.get_coords(datum)
try:
zone_tree.insert(ra, dec, datum)
except RuntimeError:
print("ERROR: Potential data corruption in " + filename)
print(
"ERROR: Check file, remove the zone directory, and re-run this program"
)
return
# prepare the save the data.
# the zone file's name
filename_no_ext = os.path.splitext(filename)[0]
# now number the (leaf) nodes
number_containers(zone_tree, zone_id=zone_id)
#plot_rects(zone_tree) # plot the nodes before the merge
zone_border_info = merge_containers_on_borders(zone_tree)
#plot_rects(zone_tree) # plot the nodes after the merge
# write out the containers that were on the border
filename = save_dir + '/' + apass.name_zone_border_file(zone_id)
save_border_info(filename, zone_border_info)
zone.save_zone_data(zone_tree, save_dir)
# save the zone -> container mapping
filename = save_dir + '/' + apass.name_zone_json_file(zone_id)
QuadTreeNode.to_file(zone_tree, filename)
def main():
"""Exports a .fredbin file to text"""
parser = argparse.ArgumentParser(
description='Merges .fred photometric files')
parser.add_argument('input',
nargs='+',
help="Input files which will be split into zonefiles")
args = parser.parse_args()
for filename in args.input:
data = fred.read_fredbin(filename)
fred.write_fredbin_txt(data, filename)
def flag_bad_data(bad_night_file, bad_field_file, filename):
"""Sets the 'use_data' entry to FALSE for any data taken on a known bad night or
bad night-field combination."""
global error_filename
try:
print("Processing %s" % (filename))
# read in the bad night/field files
bad_night_names = badfiles.read_bad_nights(bad_night_file)
bad_fields = badfiles.read_bad_night_fields(bad_field_file)
# read in the data and sort it by night
data = fred.read_fredbin(filename)
# Any given filter should ONLY set 'use_data' flags to False to avoid
# impacting other filters.
data = np.sort(data, order='night_name')
data = filter_bad_nights(data, bad_night_names)
data = np.sort(data, order='night')
data = filter_bad_night_fields(data, bad_fields)
data = filter_non_photometric_nights(data)
# restore the original order of the data
data = np.sort(data, order=['zone_id', 'node_id', 'container_id'])
# write the data
outfile = open(filename, 'w')
fred.write_fredbin(outfile, data)
except:
message = "ERROR: Failed to flag %s. Re-run in debug mode\n" % (filename)
tb = traceback.format_exc()
print(message)
with FileLock(error_filename, timeout=100, delay=0.05):
with open(error_filename, 'a') as error_file:
error_file.write(message + "\n" + str(tb) + "\n")
def upgrade_fredbin(filename):
# first try reading using standard methods
data = fred.read_fredbin(filename)
if data is not None:
print("%s is up to date" % (filename))
return None
# read using other techniques
data = read_old_fredbin(filename)
if data is None:
print("Cannot read %s " % (filename))
return None
# save the file
with open(filename, 'wb') as outfile:
print("Upgrading %s" % (filename))
fred.write_fredbin(outfile, data)
return filename
def main():
"""Plots all data in an APASS zone and optionally displays container boundaries."""
parser = argparse.ArgumentParser(
description='Plots a zone file and its data')
parser.add_argument('input',
nargs='+',
help="Zone JSON files to be plotted")
parser.add_argument('--show-containers',
default=False,
action="store_true",
help="Plot borders for containers")
args = parser.parse_args()
save_dir = os.path.dirname(os.path.realpath(args.input[0])) + "/"
inputs = args.input
for filename in inputs:
zone_id = apass.zone_from_name(filename)
zone_name = apass.name_zone(zone_id)
print "Plotting zone " + zone_name
# load the original zone data. Note, we don't restore it to the tree
zone_data_file = save_dir + apass.name_zone_file(zone_id)
zone_data = fred.read_fredbin(zone_data_file)
print("Zone file has " + str(zone_data.size) + " entries")
# load the containerized zone data
zone_container_file = save_dir + apass.name_zone_container_file(
zone_id)
zone_container_data = fred.read_fredbin(zone_container_file)
print("Zone container file has " + str(zone_container_data.size) +
" entries")
# load the zone's tree
zone_json = save_dir + apass.name_zone_json_file(zone_id)
zone_tree = QuadTreeNode.from_file(zone_json, leafClass=RectLeaf)
leaves = zone_tree.get_leaves()
total_containers = 0
for leaf in leaves:
total_containers += len(leaf.containers)
print("Zone contains a total of " + str(total_containers) +
" containers")
fig, axes = plt.subplots(1)
# plot the zone container data
for leaf in leaves:
rect = leaf.rect
x = rect.x_min
y = rect.y_min
dx = rect.x_max - x
dy = rect.y_max - y
axes.add_patch(patches.Rectangle((x, y), dx, dy, fill=False))
if args.show_containers:
plot_containers(leaf, axes)
# plot the data
dec = zone_data['dec']
ra = zone_data['ra']
plt.scatter(ra, dec)
dec = zone_container_data['dec']
ra = zone_container_data['ra']
plt.scatter(ra, dec, color="green")
plt.show()