Example #1
0
def load_tar(filename, db, config, ignored_schemas, ignored_tables):
    print "Importing data from", filename
    tar = tarfile.open(filename, 'r:bz2')
    cursor = db.cursor()
    for member in tar:
        if not member.name.startswith('mbdump/'):
            continue
        name = member.name.split('/')[1].replace('_sanitised', '')
        schema, table = parse_name(config, name)
        fulltable = fqn(schema, table)
        if schema in ignored_schemas:
            print " - Ignoring", name
            continue
        if table in ignored_tables:
            print " - Ignoring", name
            continue
        if not check_table_exists(db, schema, table):
            print " - Skipping %s (table %s does not exist)" % (name, fulltable)
            continue
        cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
        if cursor.fetchone():
            print " - Skipping %s (table %s already contains data)" % (name, fulltable)
            continue
        print " - Loading %s to %s" % (name, fulltable)
        cursor.copy_from(tar.extractfile(member), fulltable)
        db.commit()
Example #2
0
 def load_pending(self, fp):
     dump = read_psql_dump(fp, [int, str, str, int])
     for id, table, type, xid in dump:
         schema, table = parse_name(self._config, table)
         transaction = self._transactions.setdefault(xid, [])
         transaction.append((id, schema, table, type))
Example #3
0
 def load_pending(self, fp):
     dump = read_psql_dump(fp, [int, str, str, int])
     for id, table, type, xid in dump:
         schema, table = parse_name(self._config, table)
         transaction = self._transactions.setdefault(xid, [])
         transaction.append((id, schema, table, type))