def _filter_stations_parmdb(self, infile, outfile, sigma):
"""
Performs a gain outlier correction of the infile parmdb with
the corrected parmdb written to outfile.
Outliers in the gain with a distance of median of sigma times std
are replaced with the mean. The last value of the complex array
is skipped (John Swinbank: "I found it [the last value] was bad when
I hacked together some code to do this")
"""
sigma = float(sigma)
# Create copy of the input file
# delete target location
if not os.path.exists(infile):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile)
self.logger.error(message)
raise PipelineRecipeFailed(message)
delete_directory(outfile)
self.logger.debug(
"cleared target path for filtered parmdb: \n {0}".format(outfile))
# copy
shutil.copytree(infile, outfile)
self.logger.debug(
"Copied raw parmdb to target locations: \n {0}".format(infile))
# Create a local WritableParmDB
parmdb = WritableParmDB(outfile)
#get all stations in the parmdb
stations = list_stations(parmdb)
for station in stations:
self.logger.debug("Processing station {0}".format(station))
# till here implemented
polarization_data, type_pair = \
self._read_polarisation_data_and_type_from_db(parmdb, station)
corrected_data = self._swap_outliers_with_median(
polarization_data, type_pair, sigma)
#print polarization_data
self._write_corrected_data(parmdb, station, polarization_data,
corrected_data)
def _filter_stations_parmdb(self, infile, outfile, sigma):
"""
Performs a gain outlier correction of the infile parmdb with
the corrected parmdb written to outfile.
Outliers in the gain with a distance of median of sigma times std
are replaced with the mean. The last value of the complex array
is skipped (John Swinbank: "I found it [the last value] was bad when
I hacked together some code to do this")
"""
sigma = float(sigma)
# Create copy of the input file
# delete target location
if not os.path.exists(infile):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile)
self.logger.error(message)
raise PipelineRecipeFailed(message)
delete_directory(outfile)
self.logger.debug("cleared target path for filtered parmdb: \n {0}".format(
outfile))
# copy
shutil.copytree(infile, outfile)
self.logger.debug("Copied raw parmdb to target locations: \n {0}".format(
infile))
# Create a local WritableParmDB
parmdb = WritableParmDB(outfile)
#get all stations in the parmdb
stations = list_stations(parmdb)
for station in stations:
self.logger.debug("Processing station {0}".format(station))
# till here implemented
polarization_data, type_pair = \
self._read_polarisation_data_and_type_from_db(parmdb, station)
corrected_data = self._swap_outliers_with_median(polarization_data,
type_pair, sigma)
#print polarization_data
self._write_corrected_data(parmdb, station,
polarization_data, corrected_data)
def compare_two_parmdb(infile_1, infile_2, max_delta):
"""
"""
# Create copy of the input file
# delete target location
if not os.path.exists(infile_1):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile_1)
self.logger.error(message)
raise Exception(message)
if not os.path.exists(infile_2):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile_2)
self.logger.error(message)
raise Exception(message)
# copy both instrument tables (might not be needed, allows reuse of
# existing code
shutil.copytree(infile_1, infile_1 + "_copy")
shutil.copytree(infile_2, infile_2 + "_copy")
# Create a local WritableParmDB
parmdb_1 = WritableParmDB(infile_1)
parmdb_2 = WritableParmDB(infile_2)
#get all stations in the parmdb
stations_1 = list_stations(parmdb_1)
stations_2 = list_stations(parmdb_2)
try:
if len(stations_1) != len(stations_2):
print "the number of stations found in the parmdb are different!!"
print "stations_1: {0}".format(stations_1)
print "stations_2: {0}".format(stations_2)
return False
print "Number of stations in the parmdb: {0}".format(len(stations_1))
for station_1, station_2 in zip(stations_1, stations_2):
# compare the station names
if station_1 != station_2:
print "the station found in the parmdb are not the same!\n"
print "{0} != {1}".format(station_1, station_2)
return False
print "Processing station {0}".format(station_1)
# till here implemented
polarization_data_1, type_pair_1 = \
_read_polarisation_data_and_type_from_db(parmdb_1, station_1)
polarization_data_2, type_pair_2 = \
_read_polarisation_data_and_type_from_db(parmdb_2, station_1)
if type_pair_1 != type_pair_2:
print "the types found in the parmdb for station {0}are not the same!\n".format(stations_1)
print "{0} != {1}".format(type_pair_1, type_pair_2)
return False
for (pol1, data1), (pol2, data2) in zip(polarization_data_1.iteritems(),
polarization_data_2.iteritems()):
# Convert the raw data to the correct complex array type
complex_array_1 = _convert_data_to_ComplexArray(
data1, type_pair_1)
complex_array_2 = _convert_data_to_ComplexArray(
data2, type_pair_1)
# convert to magnitudes
amplitudes_1 = complex_array_1.amp[:-1]
amplitudes_2 = complex_array_2.amp[:-1]
for val_1, val_2 in zip(amplitudes_1, amplitudes_1):
if numpy.abs(val_1 - val_2) > max_delta:
print "Warning found different gains in the instrument table!"
print "station: {0}".format(station_1)
print "{0} != {1}".format(val_1, val_2)
print amplitudes_1
print amplitudes_2
return False
finally:
# remove create temp files
shutil.rmtree(infile_1 + "_copy")
shutil.rmtree(infile_2 + "_copy")
return True
def test_list_stations_get_names_with_pattern_filter(self):
list_of_names = list_stations(self.writable_parmdb, pattern = "name2*")
goal_set = ["name2"]
self.assertTrue(list_of_names == goal_set, "{0} != {1}".format(
list_of_names, goal_set))
def test_list_stations_get_names_from_writableParmdb(self):
list_of_names = list_stations(self.writable_parmdb)
goal_set = ["name1", "name2", "name3", "name4", "station1"]
self.assertTrue(list_of_names == goal_set, "{0} != {1}".format(
list_of_names, goal_set))
def test_list_stations_get_names_from_path(self):
list_of_names = list_stations("test")
goal_set = ["name1", "name2", "name3", "name4", "station1"]
self.assertTrue(list_of_names == goal_set, "{0} != {1}".format(
list_of_names, goal_set))
