def load_tar(filename, db, config, ignored_schemas, ignored_tables): print "Importing data from", filename tar = tarfile.open(filename, 'r:bz2') cursor = db.cursor() for member in tar: if not member.name.startswith('mbdump/'): continue name = member.name.split('/')[1].replace('_sanitised', '') schema, table = parse_name(config, name) fulltable = fqn(schema, table) if schema in ignored_schemas: print " - Ignoring", name continue if table in ignored_tables: print " - Ignoring", name continue if not check_table_exists(db, schema, table): print " - Skipping %s (table %s does not exist)" % (name, fulltable) continue cursor.execute("SELECT 1 FROM %s LIMIT 1" % fulltable) if cursor.fetchone(): print " - Skipping %s (table %s already contains data)" % (name, fulltable) continue print " - Loading %s to %s" % (name, fulltable) cursor.copy_from(tar.extractfile(member), fulltable) db.commit()
def process(self): cursor = self._db.cursor() stats = {} self._hook.begin(self._replication_seq) for xid in sorted(self._transactions.keys()): transaction = self._transactions[xid] #print ' - Running transaction', xid #print 'BEGIN; --', xid for id, schema, table, type in sorted(transaction): if schema == '<ignore>': continue if schema in self._ignored_schemas: continue if table in self._ignored_tables: continue fulltable = fqn(schema, table) if fulltable not in stats: stats[fulltable] = {'d': 0, 'u': 0, 'i': 0} stats[fulltable][type] += 1 keys = self._data.get((id, True), {}) values = self._data.get((id, False), {}) if type == 'd': sql = 'DELETE FROM %s' % (fulltable, ) params = [] self._hook.before_delete(table, keys) elif type == 'u': sql_values = ', '.join('%s=%%s' % i for i in values) sql = 'UPDATE %s SET %s' % (fulltable, sql_values) params = values.values() self._hook.before_update(table, keys, values) elif type == 'i': sql_columns = ', '.join(values.keys()) sql_values = ', '.join(['%s'] * len(values)) sql = 'INSERT INTO %s (%s) VALUES (%s)' % ( fulltable, sql_columns, sql_values) params = values.values() self._hook.before_insert(table, values) if type == 'd' or type == 'u': sql += ' WHERE ' + ' AND '.join( '%s%s%%s' % (i, ' IS ' if keys[i] is None else '=') for i in keys.keys()) params.extend(keys.values()) #print sql, params cursor.execute(sql, params) if type == 'd': self._hook.after_delete(table, keys) elif type == 'u': self._hook.after_update(table, keys, values) elif type == 'i': self._hook.after_insert(table, values) #print 'COMMIT; --', xid print ' - Statistics:' for table in sorted(stats.keys()): print ' * %-30s\t%d\t%d\t%d' % ( table, stats[table]['i'], stats[table]['u'], stats[table]['d']) self._hook.before_commit() self._db.commit() self._hook.after_commit()
def process(self): cursor = self._db.cursor() stats = {} self._hook.begin(self._replication_seq) for xid in sorted(self._transactions.keys()): transaction = self._transactions[xid] #print ' - Running transaction', xid #print 'BEGIN; --', xid for id, schema, table, type in sorted(transaction): if schema == '<ignore>': continue if schema in self._ignored_schemas: continue if table in self._ignored_tables: continue fulltable = fqn(schema, table) if fulltable not in stats: stats[fulltable] = {'d': 0, 'u': 0, 'i': 0} stats[fulltable][type] += 1 keys = self._data.get((id, True), {}) values = self._data.get((id, False), {}) if type == 'd': sql = 'DELETE FROM %s' % (fulltable,) params = [] self._hook.before_delete(table, keys) elif type == 'u': sql_values = ', '.join('%s=%%s' % i for i in values) sql = 'UPDATE %s SET %s' % (fulltable, sql_values) params = values.values() self._hook.before_update(table, keys, values) elif type == 'i': sql_columns = ', '.join(values.keys()) sql_values = ', '.join(['%s'] * len(values)) sql = 'INSERT INTO %s (%s) VALUES (%s)' % (fulltable, sql_columns, sql_values) params = values.values() self._hook.before_insert(table, values) if type == 'd' or type == 'u': sql += ' WHERE ' + ' AND '.join('%s%s%%s' % (i, ' IS ' if keys[i] is None else '=') for i in keys.keys()) params.extend(keys.values()) #print sql, params cursor.execute(sql, params) if type == 'd': self._hook.after_delete(table, keys) elif type == 'u': self._hook.after_update(table, keys, values) elif type == 'i': self._hook.after_insert(table, values) #print 'COMMIT; --', xid print ' - Statistics:' for table in sorted(stats.keys()): print ' * %-30s\t%d\t%d\t%d' % (table, stats[table]['i'], stats[table]['u'], stats[table]['d']) self._hook.before_commit() self._db.commit() self._hook.after_commit()