def load_tar(filename, db, config, ignored_schemas, ignored_tables):
print "Importing data from", filename
tar = tarfile.open(filename, 'r:bz2')
cursor = db.cursor()
for member in tar:
if not member.name.startswith('mbdump/'):
continue
name = member.name.split('/')[1].replace('_sanitised', '')
schema, table = parse_name(config, name)
fulltable = fqn(schema, table)
if schema in ignored_schemas:
print " - Ignoring", name
continue
if table in ignored_tables:
print " - Ignoring", name
continue
if not check_table_exists(db, schema, table):
print " - Skipping %s (table %s does not exist)" % (name, fulltable)
continue
cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable)
if cursor.fetchone():
print " - Skipping %s (table %s already contains data)" % (name, fulltable)
continue
print " - Loading %s to %s" % (name, fulltable)
cursor.copy_from(tar.extractfile(member), fulltable)
db.commit()
def process(self):
cursor = self._db.cursor()
stats = {}
self._hook.begin(self._replication_seq)
for xid in sorted(self._transactions.keys()):
transaction = self._transactions[xid]
#print ' - Running transaction', xid
#print 'BEGIN; --', xid
for id, schema, table, type in sorted(transaction):
if schema == '<ignore>':
continue
if schema in self._ignored_schemas:
continue
if table in self._ignored_tables:
continue
fulltable = fqn(schema, table)
if fulltable not in stats:
stats[fulltable] = {'d': 0, 'u': 0, 'i': 0}
stats[fulltable][type] += 1
keys = self._data.get((id, True), {})
values = self._data.get((id, False), {})
if type == 'd':
sql = 'DELETE FROM %s' % (fulltable, )
params = []
self._hook.before_delete(table, keys)
elif type == 'u':
sql_values = ', '.join('%s=%%s' % i for i in values)
sql = 'UPDATE %s SET %s' % (fulltable, sql_values)
params = values.values()
self._hook.before_update(table, keys, values)
elif type == 'i':
sql_columns = ', '.join(values.keys())
sql_values = ', '.join(['%s'] * len(values))
sql = 'INSERT INTO %s (%s) VALUES (%s)' % (
fulltable, sql_columns, sql_values)
params = values.values()
self._hook.before_insert(table, values)
if type == 'd' or type == 'u':
sql += ' WHERE ' + ' AND '.join(
'%s%s%%s' % (i, ' IS ' if keys[i] is None else '=')
for i in keys.keys())
params.extend(keys.values())
#print sql, params
cursor.execute(sql, params)
if type == 'd':
self._hook.after_delete(table, keys)
elif type == 'u':
self._hook.after_update(table, keys, values)
elif type == 'i':
self._hook.after_insert(table, values)
#print 'COMMIT; --', xid
print ' - Statistics:'
for table in sorted(stats.keys()):
print ' * %-30s\t%d\t%d\t%d' % (
table, stats[table]['i'], stats[table]['u'], stats[table]['d'])
self._hook.before_commit()
self._db.commit()
self._hook.after_commit()
def process(self):
cursor = self._db.cursor()
stats = {}
self._hook.begin(self._replication_seq)
for xid in sorted(self._transactions.keys()):
transaction = self._transactions[xid]
#print ' - Running transaction', xid
#print 'BEGIN; --', xid
for id, schema, table, type in sorted(transaction):
if schema == '<ignore>':
continue
if schema in self._ignored_schemas:
continue
if table in self._ignored_tables:
continue
fulltable = fqn(schema, table)
if fulltable not in stats:
stats[fulltable] = {'d': 0, 'u': 0, 'i': 0}
stats[fulltable][type] += 1
keys = self._data.get((id, True), {})
values = self._data.get((id, False), {})
if type == 'd':
sql = 'DELETE FROM %s' % (fulltable,)
params = []
self._hook.before_delete(table, keys)
elif type == 'u':
sql_values = ', '.join('%s=%%s' % i for i in values)
sql = 'UPDATE %s SET %s' % (fulltable, sql_values)
params = values.values()
self._hook.before_update(table, keys, values)
elif type == 'i':
sql_columns = ', '.join(values.keys())
sql_values = ', '.join(['%s'] * len(values))
sql = 'INSERT INTO %s (%s) VALUES (%s)' % (fulltable, sql_columns, sql_values)
params = values.values()
self._hook.before_insert(table, values)
if type == 'd' or type == 'u':
sql += ' WHERE ' + ' AND '.join('%s%s%%s' % (i, ' IS ' if keys[i] is None else '=') for i in keys.keys())
params.extend(keys.values())
#print sql, params
cursor.execute(sql, params)
if type == 'd':
self._hook.after_delete(table, keys)
elif type == 'u':
self._hook.after_update(table, keys, values)
elif type == 'i':
self._hook.after_insert(table, values)
#print 'COMMIT; --', xid
print ' - Statistics:'
for table in sorted(stats.keys()):
print ' * %-30s\t%d\t%d\t%d' % (table, stats[table]['i'], stats[table]['u'], stats[table]['d'])
self._hook.before_commit()
self._db.commit()
self._hook.after_commit()
