def fredbin_add_use_data(filename): """Reads in a FREDBIN file, applies modifications, saves the file. Returns the filename upon successful modification.""" print("Upgrading %s" % (filename)) # Read in the FREDBIN without the col_names = fred.fredbin_col_names[:-1] col_types = fred.fredbin_col_types[:-1] col_fmt = fred.fredbin_col_fmt[:-1] dtype = {'names': col_names, 'formats': col_types} data = np.fromfile(filename, dtype) append_col_names = ['use_data'] append_col_types = ['bool'] append_col_fmts = ['%1i'] num_rows = len(data) tmp = np.ones(num_rows) data = nprf.append_fields(data, append_col_names, [tmp], dtypes=append_col_types) fred.write_fredbin(filename, data) return filename
def purge_nights_from_file(night_names, filename): results = [] if(not os.path.isfile(filename)): return print("Processing %s" % (filename)) # At least one common night exists. Load the data. data = fred.read_fredbin(filename) if data is None: return results # first determine if this night has been stored in this data file data = np.sort(data, order='night_name') for night_name in night_names: # find the upper and lower bounds for the night l = np.searchsorted(data['night_name'], night_name, side='left') r = np.searchsorted(data['night_name'], night_name, side='right') # delete the entries data = np.delete(data, slice(l, r), axis=0) # write the remaining data to disk with open(filename, 'w') as outfile: fred.write_fredbin(outfile, data) return results
def flag_bad_data(bad_night_file, bad_field_file, filename): """Sets the 'use_data' entry to FALSE for any data taken on a known bad night or bad night-field combination.""" global error_filename try: print("Processing %s" % (filename)) # read in the bad night/field files bad_night_names = badfiles.read_bad_nights(bad_night_file) bad_fields = badfiles.read_bad_night_fields(bad_field_file) # read in the data and sort it by night data = fred.read_fredbin(filename) # Any given filter should ONLY set 'use_data' flags to False to avoid # impacting other filters. data = np.sort(data, order='night_name') data = filter_bad_nights(data, bad_night_names) data = np.sort(data, order='night') data = filter_bad_night_fields(data, bad_fields) data = filter_non_photometric_nights(data) # restore the original order of the data data = np.sort(data, order=['zone_id', 'node_id', 'container_id']) # write the data outfile = open(filename, 'w') fred.write_fredbin(outfile, data) except: message = "ERROR: Failed to flag %s. Re-run in debug mode\n" % (filename) tb = traceback.format_exc() print(message) with FileLock(error_filename, timeout=100, delay=0.05): with open(error_filename, 'a') as error_file: error_file.write(message + "\n" + str(tb) + "\n")
def upgrade_fredbin(filename): # first try reading using standard methods data = fred.read_fredbin(filename) if data is not None: print("%s is up to date" % (filename)) return None # read using other techniques data = read_old_fredbin(filename) if data is None: print("Cannot read %s " % (filename)) return None # save the file with open(filename, 'wb') as outfile: print("Upgrading %s" % (filename)) fred.write_fredbin(outfile, data) return filename
def add_fred(save_dir, filename): """Processes an APASS FRED file into zones using data contained in the tree""" print("Processing FRED file " + filename) impacted_zones = [] data_dict = build_data_dict(filename) # if there isn't any data, bail out early. if data_dict is None: return impacted_zones # write the input data to zone mapping information to a file write_mapping_info(save_dir, filename, data_dict, mode="add") # remove any data that is not for a zone del data_dict['num_fred_data'] # Write out the data being sure to lock all related files prior to opening for zone_id, data in data_dict.iteritems(): # skip zones with no data if len(data) == 0: continue zone_filename = save_dir + '/' + name_zone_file(zone_id) contrib_filename = save_dir + '/' + name_zone_contrib_file(zone_id) with FileLock(zone_filename, timeout=100, delay=0.05): with open(zone_filename, 'a+b') as outfile: write_fredbin(outfile, data) with open(contrib_filename, 'a+') as outfile: outfile.write(filename + "\n") impacted_zones.append(zone_id) print("Completed FRED file " + filename) return impacted_zones
def save_zone_data(tree, directory): """Saves zone data from the tree to the specified directory""" leaves = tree.get_leaves() zone_id = leaves[0].zone_id filename = directory + '/' + apass.name_zone_container_file(zone_id) # Extract the data from the tree data = [] for leaf in leaves: # leaf is a RectTree instance for container in leaf.containers: # container is a RectContainer instance temp = container.get_data() container.clear_data() data.extend(temp) # convert the data to a fredbin and write to file data = fred.to_fredbin(data) with open(filename, 'wb') as outfile: fred.write_fredbin(outfile, data)