def load_tar(filename, db, config, ignored_schemas, ignored_tables):
print "Importing data from", filename
tar = tarfile.open(filename, 'r:bz2')
cursor = db.cursor()
for member in tar:
if not member.name.startswith('mbdump/'):
continue
name = member.name.split('/')[1].replace('_sanitised', '')
schema, table = parse_name(config, name)
fulltable = fqn(schema, table)
if schema in ignored_schemas:
print " - Ignoring", name
continue
if table in ignored_tables:
print " - Ignoring", name
continue
if not check_table_exists(db, schema, table):
print " - Skipping %s (table %s does not exist)" % (name, fulltable)
continue
cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
if cursor.fetchone():
print " - Skipping %s (table %s already contains data)" % (name, fulltable)
continue
print " - Loading %s to %s" % (name, fulltable)
cursor.copy_from(tar.extractfile(member), fulltable)
db.commit()
def load_pending(self, fp):
dump = read_psql_dump(fp, [int, str, str, int])
for id, table, type, xid in dump:
schema, table = parse_name(self._config, table)
transaction = self._transactions.setdefault(xid, [])
transaction.append((id, schema, table, type))
def load_pending(self, fp):
dump = read_psql_dump(fp, [int, str, str, int])
for id, table, type, xid in dump:
schema, table = parse_name(self._config, table)
transaction = self._transactions.setdefault(xid, [])
transaction.append((id, schema, table, type))