def fredbin_add_use_data(filename):
"""Reads in a FREDBIN file, applies modifications, saves the file.
Returns the filename upon successful modification."""
print("Upgrading %s" % (filename))
# Read in the FREDBIN without the
col_names = fred.fredbin_col_names[:-1]
col_types = fred.fredbin_col_types[:-1]
col_fmt = fred.fredbin_col_fmt[:-1]
dtype = {'names': col_names, 'formats': col_types}
data = np.fromfile(filename, dtype)
append_col_names = ['use_data']
append_col_types = ['bool']
append_col_fmts = ['%1i']
num_rows = len(data)
tmp = np.ones(num_rows)
data = nprf.append_fields(data,
append_col_names, [tmp],
dtypes=append_col_types)
fred.write_fredbin(filename, data)
return filename
def purge_nights_from_file(night_names, filename):
results = []
if(not os.path.isfile(filename)):
return
print("Processing %s" % (filename))
# At least one common night exists. Load the data.
data = fred.read_fredbin(filename)
if data is None:
return results
# first determine if this night has been stored in this data file
data = np.sort(data, order='night_name')
for night_name in night_names:
# find the upper and lower bounds for the night
l = np.searchsorted(data['night_name'], night_name, side='left')
r = np.searchsorted(data['night_name'], night_name, side='right')
# delete the entries
data = np.delete(data, slice(l, r), axis=0)
# write the remaining data to disk
with open(filename, 'w') as outfile:
fred.write_fredbin(outfile, data)
return results
def flag_bad_data(bad_night_file, bad_field_file, filename):
"""Sets the 'use_data' entry to FALSE for any data taken on a known bad night or
bad night-field combination."""
global error_filename
try:
print("Processing %s" % (filename))
# read in the bad night/field files
bad_night_names = badfiles.read_bad_nights(bad_night_file)
bad_fields = badfiles.read_bad_night_fields(bad_field_file)
# read in the data and sort it by night
data = fred.read_fredbin(filename)
# Any given filter should ONLY set 'use_data' flags to False to avoid
# impacting other filters.
data = np.sort(data, order='night_name')
data = filter_bad_nights(data, bad_night_names)
data = np.sort(data, order='night')
data = filter_bad_night_fields(data, bad_fields)
data = filter_non_photometric_nights(data)
# restore the original order of the data
data = np.sort(data, order=['zone_id', 'node_id', 'container_id'])
# write the data
outfile = open(filename, 'w')
fred.write_fredbin(outfile, data)
except:
message = "ERROR: Failed to flag %s. Re-run in debug mode\n" % (filename)
tb = traceback.format_exc()
print(message)
with FileLock(error_filename, timeout=100, delay=0.05):
with open(error_filename, 'a') as error_file:
error_file.write(message + "\n" + str(tb) + "\n")
def upgrade_fredbin(filename):
# first try reading using standard methods
data = fred.read_fredbin(filename)
if data is not None:
print("%s is up to date" % (filename))
return None
# read using other techniques
data = read_old_fredbin(filename)
if data is None:
print("Cannot read %s " % (filename))
return None
# save the file
with open(filename, 'wb') as outfile:
print("Upgrading %s" % (filename))
fred.write_fredbin(outfile, data)
return filename
def add_fred(save_dir, filename):
"""Processes an APASS FRED file into zones using data contained in the tree"""
print("Processing FRED file " + filename)
impacted_zones = []
data_dict = build_data_dict(filename)
# if there isn't any data, bail out early.
if data_dict is None:
return impacted_zones
# write the input data to zone mapping information to a file
write_mapping_info(save_dir, filename, data_dict, mode="add")
# remove any data that is not for a zone
del data_dict['num_fred_data']
# Write out the data being sure to lock all related files prior to opening
for zone_id, data in data_dict.iteritems():
# skip zones with no data
if len(data) == 0:
continue
zone_filename = save_dir + '/' + name_zone_file(zone_id)
contrib_filename = save_dir + '/' + name_zone_contrib_file(zone_id)
with FileLock(zone_filename, timeout=100, delay=0.05):
with open(zone_filename, 'a+b') as outfile:
write_fredbin(outfile, data)
with open(contrib_filename, 'a+') as outfile:
outfile.write(filename + "\n")
impacted_zones.append(zone_id)
print("Completed FRED file " + filename)
return impacted_zones
def save_zone_data(tree, directory):
"""Saves zone data from the tree to the specified directory"""
leaves = tree.get_leaves()
zone_id = leaves[0].zone_id
filename = directory + '/' + apass.name_zone_container_file(zone_id)
# Extract the data from the tree
data = []
for leaf in leaves:
# leaf is a RectTree instance
for container in leaf.containers:
# container is a RectContainer instance
temp = container.get_data()
container.clear_data()
data.extend(temp)
# convert the data to a fredbin and write to file
data = fred.to_fredbin(data)
with open(filename, 'wb') as outfile:
fred.write_fredbin(outfile, data)