def _filter_stations_parmdb(self, infile, outfile, sigma):
        """
        Performs a gain outlier correction of the infile parmdb with
        the corrected parmdb written to outfile.
        Outliers in the gain with a distance of median of sigma times std
        are replaced with the mean. The last value of the complex array
        is skipped (John Swinbank: "I found it [the last value] was bad when 
        I hacked together some code to do this")
        """
        sigma = float(sigma)
        # Create copy of the input file
        # delete target location
        if not os.path.exists(infile):
            message = "The supplied parmdb path is not available on"
            "the filesystem: {0}".format(infile)
            self.logger.error(message)
            raise PipelineRecipeFailed(message)

        delete_directory(outfile)

        self.logger.debug(
            "cleared target path for filtered parmdb: \n {0}".format(outfile))

        # copy
        shutil.copytree(infile, outfile)
        self.logger.debug(
            "Copied raw parmdb to target locations: \n {0}".format(infile))

        # Create a local WritableParmDB
        parmdb = WritableParmDB(outfile)

        #get all stations in the parmdb
        stations = list_stations(parmdb)

        for station in stations:
            self.logger.debug("Processing station {0}".format(station))

            # till here implemented
            polarization_data, type_pair = \
               self._read_polarisation_data_and_type_from_db(parmdb, station)

            corrected_data = self._swap_outliers_with_median(
                polarization_data, type_pair, sigma)
            #print polarization_data
            self._write_corrected_data(parmdb, station, polarization_data,
                                       corrected_data)
    def _filter_stations_parmdb(self, infile, outfile, sigma):
        """
        Performs a gain outlier correction of the infile parmdb with
        the corrected parmdb written to outfile.
        Outliers in the gain with a distance of median of sigma times std
        are replaced with the mean. The last value of the complex array
        is skipped (John Swinbank: "I found it [the last value] was bad when 
        I hacked together some code to do this")
        """
        sigma = float(sigma)
        # Create copy of the input file
        # delete target location
        if not os.path.exists(infile):
            message = "The supplied parmdb path is not available on"
            "the filesystem: {0}".format(infile)
            self.logger.error(message)
            raise PipelineRecipeFailed(message)

        delete_directory(outfile)

        self.logger.debug("cleared target path for filtered parmdb: \n {0}".format(
                                outfile))

        # copy
        shutil.copytree(infile, outfile)
        self.logger.debug("Copied raw parmdb to target locations: \n {0}".format(
                                infile))

        # Create a local WritableParmDB
        parmdb = WritableParmDB(outfile)

        #get all stations in the parmdb
        stations = list_stations(parmdb)

        for station in stations:
            self.logger.debug("Processing station {0}".format(station))

            # till here implemented
            polarization_data, type_pair = \
               self._read_polarisation_data_and_type_from_db(parmdb, station)

            corrected_data = self._swap_outliers_with_median(polarization_data,
                                                  type_pair, sigma)
            #print polarization_data
            self._write_corrected_data(parmdb, station,
                                       polarization_data, corrected_data)
def compare_two_parmdb(infile_1, infile_2, max_delta):
        """
        """
        # Create copy of the input file
        # delete target location
        if not os.path.exists(infile_1):
            message = "The supplied parmdb path is not available on"
            "the filesystem: {0}".format(infile_1)
            self.logger.error(message)
            raise Exception(message)

        if not os.path.exists(infile_2):
            message = "The supplied parmdb path is not available on"
            "the filesystem: {0}".format(infile_2)
            self.logger.error(message)
            raise Exception(message)

        # copy both instrument tables (might not be needed, allows reuse of 
        # existing code
        shutil.copytree(infile_1, infile_1 + "_copy")
        shutil.copytree(infile_2, infile_2 + "_copy")

        # Create a local WritableParmDB
        parmdb_1 = WritableParmDB(infile_1)
        parmdb_2 = WritableParmDB(infile_2)

        #get all stations in the parmdb
        stations_1 = list_stations(parmdb_1)
        stations_2 = list_stations(parmdb_2)

        try:
            if len(stations_1) != len(stations_2):
                print "the number of stations found in the parmdb are different!!"
                print "stations_1: {0}".format(stations_1)
                print "stations_2: {0}".format(stations_2)
                return False
            print "Number of stations in the parmdb: {0}".format(len(stations_1))
            for station_1, station_2 in zip(stations_1, stations_2):
                # compare the station names
                if station_1 != station_2:
                    print  "the station found in the parmdb are not the same!\n"
                    print "{0} != {1}".format(station_1, station_2)

                    return False

                print "Processing station {0}".format(station_1)

                # till here implemented
                polarization_data_1, type_pair_1 = \
                   _read_polarisation_data_and_type_from_db(parmdb_1, station_1)

                polarization_data_2, type_pair_2 = \
                   _read_polarisation_data_and_type_from_db(parmdb_2, station_1)

                if type_pair_1 != type_pair_2:
                    print  "the types found in the parmdb for station {0}are not the same!\n".format(stations_1)
                    print "{0} != {1}".format(type_pair_1, type_pair_2)
                    return False

                for (pol1, data1), (pol2, data2) in zip(polarization_data_1.iteritems(),
                                     polarization_data_2.iteritems()):
                    # Convert the raw data to the correct complex array type
                    complex_array_1 = _convert_data_to_ComplexArray(
                                data1, type_pair_1)

                    complex_array_2 = _convert_data_to_ComplexArray(
                                data2, type_pair_1)

                    # convert to magnitudes
                    amplitudes_1 = complex_array_1.amp[:-1]
                    amplitudes_2 = complex_array_2.amp[:-1]

                    for val_1, val_2 in zip(amplitudes_1, amplitudes_1):
                        if numpy.abs(val_1 - val_2) > max_delta:
                            print "Warning found different gains in the instrument table!"
                            print "station: {0}".format(station_1)
                            print "{0} != {1}".format(val_1, val_2)
                            print amplitudes_1
                            print amplitudes_2
                            return False

        finally:
            # remove create temp files
            shutil.rmtree(infile_1 + "_copy")
            shutil.rmtree(infile_2 + "_copy")
        return True
 def test_list_stations_get_names_with_pattern_filter(self):
     list_of_names = list_stations(self.writable_parmdb, pattern = "name2*")
     goal_set = ["name2"]
     self.assertTrue(list_of_names == goal_set, "{0} != {1}".format(
                 list_of_names, goal_set))
 def test_list_stations_get_names_from_writableParmdb(self):
     list_of_names = list_stations(self.writable_parmdb)
     goal_set = ["name1", "name2", "name3", "name4", "station1"]
     self.assertTrue(list_of_names == goal_set, "{0} != {1}".format(
                 list_of_names, goal_set))
 def test_list_stations_get_names_from_path(self):
     list_of_names = list_stations("test")
     goal_set = ["name1", "name2", "name3", "name4", "station1"]
     self.assertTrue(list_of_names == goal_set, "{0} != {1}".format(
                 list_of_names, goal_set))