def test_read_polarisation_data_and_type_from_db(self):
#create a muck parmdb
parmdb = WritableParmDB("parmdb")
parmdb.names = ["1:1:Real:name1",
"1:1:Real:name2",
"1:1:Real:name3",
"1:1:Real:name4",
"Gain:1:1:Real:test",
"Gain:1:1:Imag:test",
"Gain:0:0:Real:test",
"Gain:0:0:Imag:test"]
station = "test"
#create sut
GainOutlierDetection = GainOutlierCorrectionWrapper()
(retrieved_data, type_pair) = GainOutlierDetection._read_polarisation_data_and_type_from_db(parmdb,
station)
#validate output!!
value_dict = {"values":[[1., 1., 1., 1., 100., 100.], [1., 1., 1., 1., 100., 100.]],
'freqs':[2],
'freqwidths':[2],
'times':[2],
'timewidths':[2]}
goal_retrieved_data = {'1:1': [value_dict, value_dict],
'0:0': [value_dict, value_dict]}
self.assertTrue(retrieved_data == goal_retrieved_data,
"Incorrect data retrieved from the parmdb: {0}".format(
retrieved_data))
goal_type_pair = ['Imag', 'Real']
self.assertTrue(type_pair == goal_type_pair,
"Incorrect data retrieved from the parmdb: {0}".format(
retrieved_data))
def test_read_polarisation_data_and_type_from_db(self):
#create a muck parmdb
parmdb = WritableParmDB("parmdb")
parmdb.names = ["1:1:Real:name1",
"1:1:Real:name2",
"1:1:Real:name3",
"1:1:Real:name4",
"Gain:1:1:Real:test",
"Gain:1:1:Imag:test",
"Gain:0:0:Real:test",
"Gain:0:0:Imag:test"]
station = "test"
#create sut
GainOutlierDetection = GainOutlierCorrectionWrapper()
(retrieved_data, type_pair) = GainOutlierDetection._read_polarisation_data_and_type_from_db(parmdb,
station)
#validate output!!
value_dict = {"values":[[1., 1., 1., 1., 100., 100.], [1., 1., 1., 1., 100., 100.]],
'freqs':[2],
'freqwidths':[2],
'times':[2],
'timewidths':[2]}
goal_retrieved_data = {'1:1': [value_dict, value_dict],
'0:0': [value_dict, value_dict]}
self.assertTrue(retrieved_data == goal_retrieved_data,
"Incorrect data retrieved from the parmdb: {0}".format(
retrieved_data))
goal_type_pair = ['Imag', 'Real']
self.assertTrue(type_pair == goal_type_pair,
"Incorrect data retrieved from the parmdb: {0}".format(
retrieved_data))
def test_write_corrected_data_does_not_contain_pol(self):
name = "test"
station = "station"
GainOutlierDetection = GainOutlierCorrectionWrapper()
input_polarization_data = {
"unknownPolarisation": [{
'freqs': [11],
'freqwidths': [12],
'times': [13],
'timewidths': [14]
}]
}
input_corected_data = {
"pol1": RealImagArray([[1], [1]], [[2], [2]]),
"pol2": RealImagArray([[3], [3]], [[4], [4]])
}
# This object will be taken from the fixture: it is a recorder muck
parmdb = WritableParmDB("parmdb")
# call function
GainOutlierDetection = GainOutlierCorrectionWrapper()
self.assertRaises(PipelineRecipeFailed,
GainOutlierDetection._write_corrected_data, parmdb,
station, input_polarization_data,
input_corected_data)
def test_read_polarisation_data_and_type_from_db_invalid_type_pair(self):
#create a muck parmdb
parmdb = WritableParmDB("parmdb")
parmdb.names = [
"1:1:Real:name1", "1:1:Real:name2", "1:1:Real:name3",
"1:1:Real:name4", "Gain:1:1:Real:test", "Gain:1:1:Imag:test",
"Gain:0:0:incorrect_type:test", "Gain:0:0:Imag:test"
]
station = "test"
GainOutlierDetection = GainOutlierCorrectionWrapper()
# unknown datatype should throw an exception
self.assertRaises(
PipelineRecipeFailed,
GainOutlierDetection._read_polarisation_data_and_type_from_db,
parmdb, station)
def test_read_polarisation_data_and_type_from_db_invalid_type_pair(self):
#create a muck parmdb
parmdb = WritableParmDB("parmdb")
parmdb.names = ["1:1:Real:name1",
"1:1:Real:name2",
"1:1:Real:name3",
"1:1:Real:name4",
"Gain:1:1:Real:test",
"Gain:1:1:Imag:test",
"Gain:0:0:incorrect_type:test",
"Gain:0:0:Imag:test"]
station = "test"
GainOutlierDetection = GainOutlierCorrectionWrapper()
# unknown datatype should throw an exception
self.assertRaises(PipelineRecipeFailed, GainOutlierDetection._read_polarisation_data_and_type_from_db,
parmdb, station)
def _filter_stations_parmdb(self, infile, outfile, sigma):
"""
Performs a gain outlier correction of the infile parmdb with
the corrected parmdb written to outfile.
Outliers in the gain with a distance of median of sigma times std
are replaced with the mean. The last value of the complex array
is skipped (John Swinbank: "I found it [the last value] was bad when
I hacked together some code to do this")
"""
sigma = float(sigma)
# Create copy of the input file
# delete target location
if not os.path.exists(infile):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile)
self.logger.error(message)
raise PipelineRecipeFailed(message)
delete_directory(outfile)
self.logger.debug(
"cleared target path for filtered parmdb: \n {0}".format(outfile))
# copy
shutil.copytree(infile, outfile)
self.logger.debug(
"Copied raw parmdb to target locations: \n {0}".format(infile))
# Create a local WritableParmDB
parmdb = WritableParmDB(outfile)
#get all stations in the parmdb
stations = list_stations(parmdb)
for station in stations:
self.logger.debug("Processing station {0}".format(station))
# till here implemented
polarization_data, type_pair = \
self._read_polarisation_data_and_type_from_db(parmdb, station)
corrected_data = self._swap_outliers_with_median(
polarization_data, type_pair, sigma)
#print polarization_data
self._write_corrected_data(parmdb, station, polarization_data,
corrected_data)
def test_write_corrected_data(self):
# define input data
name = "test"
station = "station"
GainOutlierDetection = GainOutlierCorrectionWrapper()
input_polarization_data = {
"pol1": [{
'freqs': [11],
'freqwidths': [12],
'times': [13],
'timewidths': [14]
}]
}
input_corected_data = {
"pol1": RealImagArray([[1], [1]], [[2], [2]]),
"pol22": RealImagArray([[3], [3]], [[4], [4]])
}
# This object will be taken from the fixture: it is a recorder muck
parmdb = WritableParmDB("parmdb")
# call function
GainOutlierDetection = GainOutlierCorrectionWrapper()
GainOutlierDetection._write_corrected_data(parmdb, station,
input_polarization_data,
input_corected_data)
# test output: (the calls to parmdb)
# there is one polarization, containing a single complex array
# when writing this should result in, 1 times 2 function calls
# first delete the REAL entry
expected = ['deleteValues', ['Gain:pol1:Real:station']]
self.assertTrue(
parmdb.called_functions_and_parameters[0] == expected,
"result({0}) != expected({1})".format(
parmdb.called_functions_and_parameters[0], expected))
# then the new values should be added, with the correct values
expected = [
'addValues',
[
'Gain:pol1:Real:station',
numpy.array([[1.], [1.]], ), 11, 11 + 12, 13, 13 + 2 * 14, True
]
] #stat + steps*size
# Now scan the argument array: for numpy use special compare function
for left, right in zip(parmdb.called_functions_and_parameters[1][1],
expected[1]):
error_message = "\nresult({0}) != \nexpected({1}) \n"\
"-> {2} != {3}".format(
parmdb.called_functions_and_parameters[1], expected,
left, right)
try:
if not left == right:
self.assertTrue(False, error_message)
except ValueError:
if not numpy.array_equal(left, right):
self.assertTrue(False, error_message)
# now delete the imag entry: Rememder these are on the 2nd and 3th array
# position
expected = ['deleteValues', ['Gain:pol1:Imag:station']]
self.assertTrue(
parmdb.called_functions_and_parameters[2] == expected,
"result({0}) != expected({1})".format(
parmdb.called_functions_and_parameters[2], expected))
# then the new values should be added, with the correct values
expected = [
'addValues',
[
'Gain:pol1:Imag:station',
numpy.array([[2.], [2.]], ), 11, 11 + 12, 13, 13 + 2 * 14, True
]
] #stat + steps*size
# Now scan the argument array: for numpy use special compare function
for left, right in zip(parmdb.called_functions_and_parameters[3][1],
expected[1]):
error_message = "\nresult({0}) != \nexpected({1}) \n"\
"-> {2} != {3}".format(
parmdb.called_functions_and_parameters[3], expected,
left, right)
try:
if not left == right:
self.assertTrue(False, error_message)
except ValueError:
if not numpy.array_equal(left, right):
self.assertTrue(False, error_message)
def compare_two_parmdb(infile_1, infile_2, max_delta):
"""
"""
# Create copy of the input file
# delete target location
if not os.path.exists(infile_1):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile_1)
self.logger.error(message)
raise Exception(message)
if not os.path.exists(infile_2):
message = "The supplied parmdb path is not available on"
"the filesystem: {0}".format(infile_2)
self.logger.error(message)
raise Exception(message)
# copy both instrument tables (might not be needed, allows reuse of
# existing code
shutil.copytree(infile_1, infile_1 + "_copy")
shutil.copytree(infile_2, infile_2 + "_copy")
# Create a local WritableParmDB
parmdb_1 = WritableParmDB(infile_1)
parmdb_2 = WritableParmDB(infile_2)
#get all stations in the parmdb
stations_1 = list_stations(parmdb_1)
stations_2 = list_stations(parmdb_2)
try:
if len(stations_1) != len(stations_2):
print "the number of stations found in the parmdb are different!!"
print "stations_1: {0}".format(stations_1)
print "stations_2: {0}".format(stations_2)
return False
print "Number of stations in the parmdb: {0}".format(len(stations_1))
for station_1, station_2 in zip(stations_1, stations_2):
# compare the station names
if station_1 != station_2:
print "the station found in the parmdb are not the same!\n"
print "{0} != {1}".format(station_1, station_2)
return False
print "Processing station {0}".format(station_1)
# till here implemented
polarization_data_1, type_pair_1 = \
_read_polarisation_data_and_type_from_db(parmdb_1, station_1)
polarization_data_2, type_pair_2 = \
_read_polarisation_data_and_type_from_db(parmdb_2, station_1)
if type_pair_1 != type_pair_2:
print "the types found in the parmdb for station {0}are not the same!\n".format(stations_1)
print "{0} != {1}".format(type_pair_1, type_pair_2)
return False
for (pol1, data1), (pol2, data2) in zip(polarization_data_1.iteritems(),
polarization_data_2.iteritems()):
# Convert the raw data to the correct complex array type
complex_array_1 = _convert_data_to_ComplexArray(
data1, type_pair_1)
complex_array_2 = _convert_data_to_ComplexArray(
data2, type_pair_1)
# convert to magnitudes
amplitudes_1 = complex_array_1.amp[:-1]
amplitudes_2 = complex_array_2.amp[:-1]
for val_1, val_2 in zip(amplitudes_1, amplitudes_1):
if numpy.abs(val_1 - val_2) > max_delta:
print "Warning found different gains in the instrument table!"
print "station: {0}".format(station_1)
print "{0} != {1}".format(val_1, val_2)
print amplitudes_1
print amplitudes_2
return False
finally:
# remove create temp files
shutil.rmtree(infile_1 + "_copy")
shutil.rmtree(infile_2 + "_copy")
return True