Пример #1
0
    def test_safe_name(self):
        "Test safe_name database name sanitiser."

        self.assertEqual(dbutil.safe_name('Fred'), 'Fred')
        self.assertEqual(dbutil.safe_name('Fred_123456789'), 'Fred_123456789')
        self.assertEqual(dbutil.safe_name('Fred!@%&***'), 'Fred')
        self.assertEqual(dbutil.safe_name('Fred;drop postgres;'),
                         'Freddroppostgres')
Пример #2
0
    def test_safe_name(self):
        "Test safe_name database name sanitiser."

        self.assertEqual(dbutil.safe_name('Fred'), 'Fred')
        self.assertEqual(dbutil.safe_name('Fred_123456789'), 'Fred_123456789')
        self.assertEqual(dbutil.safe_name('Fred!@%&***'), 'Fred')
        self.assertEqual(dbutil.safe_name('Fred;drop postgres;'),
                         'Freddroppostgres')
Пример #3
0
    def __init__(self, db1, db2, schema1='public', schema2='public',
                 verbosity=0, output=sys.stdout):
        """
        Positional Arguments:
            db1, db2: Connections to the databases to be compared.

        Keyword Arguments:
            schema1: The schema to be used for the first database (db1),
                defaults to 'public'.
            schema2: The schema to be used for the second database (db2),
                defaults to 'public'.
            verbosity: Amount of output generated if a difference is detected.
                0 -- No output, just the return value.
                1 -- Missing columns, mismatched primary keys,
                     one line notification of table content differences.
                2 -- As above, but prints the details of the first
                     MAX_DIFFERENCES content differences in each table.
                3 -- As above, but prints the details of all differences.
                     Defaults to 0.
            output: Where the output goes. This is assumed to be a file object.
                Defaults to sys.stdout.
        """

        # Set autocommit mode on the connections; retain the old settings.
        self.old_autocommit = (db1.autocommit, db2.autocommit)
        db1.autocommit = True
        db2.autocommit = True

        # Sanitise the schema names, just in case.
        self.schema1 = dbutil.safe_name(schema1)
        self.schema2 = dbutil.safe_name(schema2)

        # Wrap the connections to gain access to database structure queries.
        self.db1 = ComparisonWrapper(db1, self.schema1)
        self.db2 = ComparisonWrapper(db2, self.schema2)

        # Get the database names...
        self.db1_name = self.db1.database_name()
        self.db2_name = self.db2.database_name()

        # and qualify with the schema names if they are not 'public'
        if self.schema1 != 'public':
            self.db1_name = self.schema1 + '.' + self.db1_name
        if self.schema2 != 'public':
            self.db2_name = self.schema2 + '.' + self.db2_name

        self.report = Reporter(self.db1_name, self.db2_name,
                               verbosity, output)
Пример #4
0
    def __init__(self, db1, db2, schema1="public", schema2="public", verbosity=0, output=sys.stdout):
        """
        Positional Arguments:
            db1, db2: Connections to the databases to be compared.

        Keyword Arguments:
            schema1: The schema to be used for the first database (db1),
                defaults to 'public'.
            schema2: The schema to be used for the second database (db2),
                defaults to 'public'.
            verbosity: Amount of output generated if a difference is detected.
                0 -- No output, just the return value.
                1 -- Missing columns, mismatched primary keys,
                     one line notification of table content differences.
                2 -- As above, but prints the details of the first
                     MAX_DIFFERENCES content differences in each table.
                3 -- As above, but prints the details of all differences.
                     Defaults to 0.
            output: Where the output goes. This is assumed to be a file object.
                Defaults to sys.stdout.
        """

        # Set autocommit mode on the connections; retain the old settings.
        self.old_autocommit = (db1.autocommit, db2.autocommit)
        db1.autocommit = True
        db2.autocommit = True

        # Sanitise the schema names, just in case.
        self.schema1 = dbutil.safe_name(schema1)
        self.schema2 = dbutil.safe_name(schema2)

        # Wrap the connections to gain access to database structure queries.
        self.db1 = ComparisonWrapper(db1, self.schema1)
        self.db2 = ComparisonWrapper(db2, self.schema2)

        # Get the database names...
        self.db1_name = self.db1.database_name()
        self.db2_name = self.db2.database_name()

        # and qualify with the schema names if they are not 'public'
        if self.schema1 != "public":
            self.db1_name = self.schema1 + "." + self.db1_name
        if self.schema2 != "public":
            self.db2_name = self.schema2 + "." + self.db2_name

        self.report = Reporter(self.db1_name, self.db2_name, verbosity, output)
Пример #5
0
    def __init__(self, db1, db2, schema1, schema2):
        """
        Positional Arguments:
            db1, db2: Connections to the databases to be compared.

        Keyword Arguments:
            schema1: The schema to be used for the first database (db1)
            schema2: The schema to be used for the second database (db2)
        """

        # Set autocommit mode on the connections; retain the old settings.
        self.old_autocommit = (db1.autocommit, db2.autocommit)
        db1.autocommit = True
        db2.autocommit = True

        # Sanitise the schema names, just in case.
        self.schema1 = dbutil.safe_name(schema1)
        self.schema2 = dbutil.safe_name(schema2)

        # Wrap the connections to gain access to database structure queries.
        self.db1 = ComparisonWrapper(db1, self.schema1)
        self.db2 = ComparisonWrapper(db2, self.schema2)

        # Get the database names...
        self.db1_name = self.db1.database_name()
        self.db2_name = self.db2.database_name()

        # and qualify with the schema names if they are not 'public'
        if self.schema1 != 'public':
            self.db1_name = self.schema1 + '.' + self.db1_name
        if self.schema2 != 'public':
            self.db2_name = self.schema2 + '.' + self.db2_name

        # Set input, expected and output directores
        # Not used yet
        module = "tilecompare"
        suite = "TileCompare"
        self.input_dir = dbutil.input_directory(module, suite)
        self.output_dir = dbutil.output_directory(module, suite)
        self.expected_dir = dbutil.expected_directory(module, suite)
Пример #6
0
    def __init__(self, db1, db2, schema1, schema2):
        """
        Positional Arguments:
            db1, db2: Connections to the databases to be compared.

        Keyword Arguments:
            schema1: The schema to be used for the first database (db1)
            schema2: The schema to be used for the second database (db2)
        """

        # Set autocommit mode on the connections; retain the old settings.
        self.old_autocommit = (db1.autocommit, db2.autocommit)
        db1.autocommit = True
        db2.autocommit = True

        # Sanitise the schema names, just in case.
        self.schema1 = dbutil.safe_name(schema1)
        self.schema2 = dbutil.safe_name(schema2)

        # Wrap the connections to gain access to database structure queries.
        self.db1 = ComparisonWrapper(db1, self.schema1)
        self.db2 = ComparisonWrapper(db2, self.schema2)

        # Get the database names...
        self.db1_name = self.db1.database_name()
        self.db2_name = self.db2.database_name()

        # and qualify with the schema names if they are not 'public'
        if self.schema1 != 'public':
            self.db1_name = self.schema1 + '.' + self.db1_name
        if self.schema2 != 'public':
            self.db2_name = self.schema2 + '.' + self.db2_name

        # Set input, expected and output directores
        # Not used yet
        module = "tilecompare"
        suite = "TileCompare"
        self.input_dir = dbutil.input_directory(module, suite)
        self.output_dir = dbutil.output_directory(module, suite)
        self.expected_dir = dbutil.expected_directory(module, suite)
Пример #7
0
def compare_tables(db1, db2, table, schema1='public', schema2='public',
                   ignore_columns=None, verbosity=0, output=sys.stdout):
    """Compares tables from two databases, returns True if identical.

    This function compares the tables named 'table' in two databases.
    It checks that non-ignored columns are present in both tables, and it
    checks that the contents match. It does not care about column ordering,
    but does use the primary key to order the rows for the content check.
    It will regard different primary keys as a significant difference,
    and will report an error if either table does not have a primary key.

    Using the primary key for ordering means that the results are dependent
    on the order of insertion for records which have an auto-generated key.
    This is a limitation of the current implementation.

    Preconditions: db1 and db2 are open database connections. These are
        assumed to be psycopg2 connections to PostgreSQL databases.
        table is the name of a table present in both databases. This
        table has a primary key in both databases.

    Positional Arguments:
        db1, db2: Connections to the databases to be compared.
        table: The name of the table to be compared.

    Keyword Arguments:
        schema1: The schema to be used for the first database (db1), defaults
            to 'public'.
        schema2: The schema to be used for the second database (db2), defaults
            to 'public'.
        ignore_columns: A list (or other Python iterable) of columns to be
            ignored. These may optionally be qualified by table e.g.
            'dataset.datetime_processed'. Qualified column names for tables
            other than table may be present in this list, but will have
            no effect. The contents of the ignored columns will not be
            compared, and the comparison will not care if they are only in the
            table in one database and not the other. Defaults to an empty list.
        verbosity: Amount of output generated if a difference is detected.
            0 -- No output, just the return value.
            1 -- Missing columns, mismatched primary keys,
                 one line notification of table content differences.
            2 -- As above, but prints the details of the first MAX_DIFFERENCES
                 content differences in each table.
            3 -- As above, but prints the details of all differences.
            Defaults to 0.
        output: Where the output goes. This is assumed to be a file object.
            Defaults to sys.stdout.

    Return Value: Returns True if the tables 'table' are identical in
        both databases, excepting columns specifed as ignored by the
        arguments. Returns False otherwise.

    Postconditions: This function should have no side effects, except for
        the output generated if verbosity is greater than 0.
    """

    if ignore_columns is None:
        ignore_columns = []

    table = dbutil.safe_name(table)

    comparison = Comparison(db1, db2, schema1=schema1, schema2=schema2,
                            verbosity=verbosity, output=output)

    assert comparison.db1.table_exists(table), \
        ("Could not find table '%s' in database '%s'." %
         (table, comparison.db1_name))
    assert comparison.db2.table_exists(table), \
        ("Could not find table '%s' in database '%s'." %
         (table, comparison.db2_name))

    tables_match = comparison.compare_tables(table, ignore_columns)

    comparison.restore_autocommit()

    return tables_match
Пример #8
0
    def __init__(self, conn, default_schema='public'):

        self.default_schema = dbutil.safe_name(default_schema)
        dbutil.ConnectionWrapper.__init__(self, conn)
Пример #9
0
def compare_tables(
    db1, db2, table, schema1="public", schema2="public", ignore_columns=None, verbosity=0, output=sys.stdout
):
    """Compares tables from two databases, returns True if identical.

    This function compares the tables named 'table' in two databases.
    It checks that non-ignored columns are present in both tables, and it
    checks that the contents match. It does not care about column ordering,
    but does use the primary key to order the rows for the content check.
    It will regard different primary keys as a significant difference,
    and will report an error if either table does not have a primary key.

    Using the primary key for ordering means that the results are dependent
    on the order of insertion for records which have an auto-generated key.
    This is a limitation of the current implementation.

    Preconditions: db1 and db2 are open database connections. These are
        assumed to be psycopg2 connections to PostgreSQL databases.
        table is the name of a table present in both databases. This
        table has a primary key in both databases.

    Positional Arguments:
        db1, db2: Connections to the databases to be compared.
        table: The name of the table to be compared.

    Keyword Arguments:
        schema1: The schema to be used for the first database (db1), defaults
            to 'public'.
        schema2: The schema to be used for the second database (db2), defaults
            to 'public'.
        ignore_columns: A list (or other Python iterable) of columns to be
            ignored. These may optionally be qualified by table e.g.
            'dataset.datetime_processed'. Qualified column names for tables
            other than table may be present in this list, but will have
            no effect. The contents of the ignored columns will not be
            compared, and the comparison will not care if they are only in the
            table in one database and not the other. Defaults to an empty list.
        verbosity: Amount of output generated if a difference is detected.
            0 -- No output, just the return value.
            1 -- Missing columns, mismatched primary keys,
                 one line notification of table content differences.
            2 -- As above, but prints the details of the first MAX_DIFFERENCES
                 content differences in each table.
            3 -- As above, but prints the details of all differences.
            Defaults to 0.
        output: Where the output goes. This is assumed to be a file object.
            Defaults to sys.stdout.

    Return Value: Returns True if the tables 'table' are identical in
        both databases, excepting columns specifed as ignored by the
        arguments. Returns False otherwise.

    Postconditions: This function should have no side effects, except for
        the output generated if verbosity is greater than 0.
    """

    if ignore_columns is None:
        ignore_columns = []

    table = dbutil.safe_name(table)

    comparison = Comparison(db1, db2, schema1=schema1, schema2=schema2, verbosity=verbosity, output=output)

    assert comparison.db1.table_exists(table), "Could not find table '%s' in database '%s'." % (
        table,
        comparison.db1_name,
    )
    assert comparison.db2.table_exists(table), "Could not find table '%s' in database '%s'." % (
        table,
        comparison.db2_name,
    )

    tables_match = comparison.compare_tables(table, ignore_columns)

    comparison.restore_autocommit()

    return tables_match
Пример #10
0
    def __init__(self, conn, default_schema="public"):

        self.default_schema = dbutil.safe_name(default_schema)
        dbutil.ConnectionWrapper.__init__(self, conn)