def _filter_stations_parmdb(self, infile, outfile, sigma): """ Performs a gain outlier correction of the infile parmdb with the corrected parmdb written to outfile. Outliers in the gain with a distance of median of sigma times std are replaced with the mean. The last value of the complex array is skipped (John Swinbank: "I found it [the last value] was bad when I hacked together some code to do this") """ sigma = float(sigma) # Create copy of the input file # delete target location if not os.path.exists(infile): message = "The supplied parmdb path is not available on" "the filesystem: {0}".format(infile) self.logger.error(message) raise PipelineRecipeFailed(message) delete_directory(outfile) self.logger.debug( "cleared target path for filtered parmdb: \n {0}".format(outfile)) # copy shutil.copytree(infile, outfile) self.logger.debug( "Copied raw parmdb to target locations: \n {0}".format(infile)) # Create a local WritableParmDB parmdb = WritableParmDB(outfile) #get all stations in the parmdb stations = list_stations(parmdb) for station in stations: self.logger.debug("Processing station {0}".format(station)) # till here implemented polarization_data, type_pair = \ self._read_polarisation_data_and_type_from_db(parmdb, station) corrected_data = self._swap_outliers_with_median( polarization_data, type_pair, sigma) #print polarization_data self._write_corrected_data(parmdb, station, polarization_data, corrected_data)
def _filter_stations_parmdb(self, infile, outfile, sigma): """ Performs a gain outlier correction of the infile parmdb with the corrected parmdb written to outfile. Outliers in the gain with a distance of median of sigma times std are replaced with the mean. The last value of the complex array is skipped (John Swinbank: "I found it [the last value] was bad when I hacked together some code to do this") """ sigma = float(sigma) # Create copy of the input file # delete target location if not os.path.exists(infile): message = "The supplied parmdb path is not available on" "the filesystem: {0}".format(infile) self.logger.error(message) raise PipelineRecipeFailed(message) delete_directory(outfile) self.logger.debug("cleared target path for filtered parmdb: \n {0}".format( outfile)) # copy shutil.copytree(infile, outfile) self.logger.debug("Copied raw parmdb to target locations: \n {0}".format( infile)) # Create a local WritableParmDB parmdb = WritableParmDB(outfile) #get all stations in the parmdb stations = list_stations(parmdb) for station in stations: self.logger.debug("Processing station {0}".format(station)) # till here implemented polarization_data, type_pair = \ self._read_polarisation_data_and_type_from_db(parmdb, station) corrected_data = self._swap_outliers_with_median(polarization_data, type_pair, sigma) #print polarization_data self._write_corrected_data(parmdb, station, polarization_data, corrected_data)
def compare_two_parmdb(infile_1, infile_2, max_delta): """ """ # Create copy of the input file # delete target location if not os.path.exists(infile_1): message = "The supplied parmdb path is not available on" "the filesystem: {0}".format(infile_1) self.logger.error(message) raise Exception(message) if not os.path.exists(infile_2): message = "The supplied parmdb path is not available on" "the filesystem: {0}".format(infile_2) self.logger.error(message) raise Exception(message) # copy both instrument tables (might not be needed, allows reuse of # existing code shutil.copytree(infile_1, infile_1 + "_copy") shutil.copytree(infile_2, infile_2 + "_copy") # Create a local WritableParmDB parmdb_1 = WritableParmDB(infile_1) parmdb_2 = WritableParmDB(infile_2) #get all stations in the parmdb stations_1 = list_stations(parmdb_1) stations_2 = list_stations(parmdb_2) try: if len(stations_1) != len(stations_2): print "the number of stations found in the parmdb are different!!" print "stations_1: {0}".format(stations_1) print "stations_2: {0}".format(stations_2) return False print "Number of stations in the parmdb: {0}".format(len(stations_1)) for station_1, station_2 in zip(stations_1, stations_2): # compare the station names if station_1 != station_2: print "the station found in the parmdb are not the same!\n" print "{0} != {1}".format(station_1, station_2) return False print "Processing station {0}".format(station_1) # till here implemented polarization_data_1, type_pair_1 = \ _read_polarisation_data_and_type_from_db(parmdb_1, station_1) polarization_data_2, type_pair_2 = \ _read_polarisation_data_and_type_from_db(parmdb_2, station_1) if type_pair_1 != type_pair_2: print "the types found in the parmdb for station {0}are not the same!\n".format(stations_1) print "{0} != {1}".format(type_pair_1, type_pair_2) return False for (pol1, data1), (pol2, data2) in zip(polarization_data_1.iteritems(), polarization_data_2.iteritems()): # Convert the raw data to the correct complex array type complex_array_1 = _convert_data_to_ComplexArray( data1, type_pair_1) complex_array_2 = _convert_data_to_ComplexArray( data2, type_pair_1) # convert to magnitudes amplitudes_1 = complex_array_1.amp[:-1] amplitudes_2 = complex_array_2.amp[:-1] for val_1, val_2 in zip(amplitudes_1, amplitudes_1): if numpy.abs(val_1 - val_2) > max_delta: print "Warning found different gains in the instrument table!" print "station: {0}".format(station_1) print "{0} != {1}".format(val_1, val_2) print amplitudes_1 print amplitudes_2 return False finally: # remove create temp files shutil.rmtree(infile_1 + "_copy") shutil.rmtree(infile_2 + "_copy") return True
def test_list_stations_get_names_with_pattern_filter(self): list_of_names = list_stations(self.writable_parmdb, pattern = "name2*") goal_set = ["name2"] self.assertTrue(list_of_names == goal_set, "{0} != {1}".format( list_of_names, goal_set))
def test_list_stations_get_names_from_writableParmdb(self): list_of_names = list_stations(self.writable_parmdb) goal_set = ["name1", "name2", "name3", "name4", "station1"] self.assertTrue(list_of_names == goal_set, "{0} != {1}".format( list_of_names, goal_set))
def test_list_stations_get_names_from_path(self): list_of_names = list_stations("test") goal_set = ["name1", "name2", "name3", "name4", "station1"] self.assertTrue(list_of_names == goal_set, "{0} != {1}".format( list_of_names, goal_set))