def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
q = "select * from londiste.local_add_table(%s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
dst_db.commit()
def upgrade(self, dbname, db):
"""Upgrade all schemas in single db."""
curs = db.cursor()
ignore = {}
for schema, ver, fn, recheck_func in version_list:
# skip schema?
if schema in ignore:
continue
if not skytools.exists_schema(curs, schema):
ignore[schema] = 1
continue
# new enough?
if check_version(curs, schema, ver, recheck_func):
continue
# too old schema, no way to upgrade
if fn is None:
self.log.info('%s: Cannot upgrade %s, too old version', dbname, schema)
ignore[schema] = 1
continue
curs = db.cursor()
curs.execute('begin')
skytools.installer_apply_file(db, fn, self.log)
curs.execute('commit')
def upgrade(self, dbname, db):
"""Upgrade all schemas in single db."""
curs = db.cursor()
ignore = {}
for schema, ver, fn, recheck_func in version_list:
# skip schema?
if schema in ignore:
continue
if not skytools.exists_schema(curs, schema):
ignore[schema] = 1
continue
# new enough?
ok, oldver = check_version(curs, schema, ver, recheck_func, self.options.force)
if ok:
continue
# too old schema, no way to upgrade
if fn is None:
self.log.info('%s: Cannot upgrade %s, too old version', dbname, schema)
ignore[schema] = 1
continue
if self.options.not_really:
self.log.info ("%s: Would upgrade '%s' version %s to %s", dbname, schema, oldver, ver)
continue
curs = db.cursor()
curs.execute('begin')
self.log.info("%s: Upgrading '%s' version %s to %s", dbname, schema, oldver, ver)
skytools.installer_apply_file(db, fn, self.log)
curs.execute('commit')
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
# actual table registration
q = "select * from londiste.local_add_table(%s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
if self.options.expect_sync:
q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
if self.options.copy_condition:
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
attrs = {'copy_condition': self.options.copy_condition}
enc_attrs = skytools.db_urlencode(attrs)
self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs])
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs['handler'] = hstr
p.add(tgargs)
# actual table registration
q = "select * from londiste.local_add_table(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs])
if self.options.expect_sync:
q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
else:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.copy_condition:
attrs['copy_condition'] = self.options.copy_condition
if attrs:
enc_attrs = skytools.db_urlencode(attrs)
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs])
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs['handler'] = hstr
p.add(tgargs)
# actual table registration
q = "select * from londiste.local_add_table(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs])
if self.options.expect_sync:
q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
else:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.copy_condition:
attrs['copy_condition'] = self.options.copy_condition
if attrs:
enc_attrs = skytools.db_urlencode(attrs)
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs])
dst_db.commit()
def upgrade(self, db):
curs = db.cursor()
for schema, ver, sql, recheck_fn in version_list:
if not skytools.exists_schema(curs, schema):
continue
if check_version(curs, schema, ver, recheck_fn):
continue
fn = "upgrade/final/%s" % sql
skytools.installer_apply_file(db, fn, self.log)
def upgrade(self, db):
curs = db.cursor()
for schema, ver, sql, recheck_fn in version_list:
if not skytools.exists_schema(curs, schema):
continue
if check_version(curs, schema, ver, recheck_fn):
continue
fn = "upgrade/final/%s" % sql
skytools.installer_apply_file(db, fn, self.log)
def _ccache(self, cname, q, req_schema = None):
if not self.db:
return []
# check if schema exists
if req_schema:
k = "schema_exists_%s" % req_schema
ok = self.comp_cache.get(k)
if ok is None:
curs = self.db.cursor()
ok = skytools.exists_schema(curs, req_schema)
self.comp_cache[k] = ok
if not ok:
return []
# actual completion
clist = self.comp_cache.get(cname)
if clist is None:
curs = self.db.cursor()
curs.execute(q)
clist = [r[0] for r in curs.fetchall()]
self.comp_cache[cname] = clist
return clist
def _ccache(self, cname, q, req_schema = None):
if not self.db:
return []
# check if schema exists
if req_schema:
k = "schema_exists_%s" % req_schema
ok = self.comp_cache.get(k)
if ok is None:
curs = self.db.cursor()
ok = skytools.exists_schema(curs, req_schema)
self.comp_cache[k] = ok
if not ok:
return []
# actual completion
clist = self.comp_cache.get(cname)
if clist is None:
curs = self.db.cursor()
curs.execute(q)
clist = [r[0] for r in curs.fetchall()]
self.comp_cache[cname] = clist
return clist
def upgrade(self, dbname, db):
"""Upgrade all schemas in single db."""
curs = db.cursor()
ignore = {}
for schema, ver, fn, recheck_func in version_list:
# skip schema?
if schema in ignore:
continue
if not skytools.exists_schema(curs, schema):
ignore[schema] = 1
continue
# new enough?
ok, oldver = check_version(curs, schema, ver, recheck_func)
if ok:
continue
# too old schema, no way to upgrade
if fn is None:
self.log.info('%s: Cannot upgrade %s, too old version', dbname,
schema)
ignore[schema] = 1
continue
if self.options.not_really:
self.log.info("%s: Would upgrade '%s' version %s to %s",
dbname, schema, oldver, ver)
continue
curs = db.cursor()
curs.execute('begin')
self.log.info("%s: Upgrading '%s' version %s to %s", dbname,
schema, oldver, ver)
skytools.installer_apply_file(db, fn, self.log)
curs.execute('commit')
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls):
# use full names
tbl = skytools.fq_name(tbl)
dest_table = self.options.dest_table or tbl
dest_table = skytools.fq_name(dest_table)
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
if dest_table == tbl:
desc = tbl
else:
desc = "%s(%s)" % (tbl, dest_table)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % desc)
else:
src_dest_table = src_tbls[tbl]['dest_table']
if not skytools.exists_table(src_curs, src_dest_table):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table %s missing on provider, cannot create, skipping' % desc)
return
schema = skytools.fq_name_parts(dest_table)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, src_dest_table)
src_db.commit()
# create, using rename logic only when necessary
newname = None
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs, create_flags, log = self.log, new_table_name = newname)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
if self.options.no_merge:
tgargs.append('no_merge')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table)
attrs['handler'] = hstr
p.add(tgargs)
if self.options.find_copy_node:
attrs['copy_node'] = '?'
elif self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
if self.options.expect_sync:
tgargs.append('expect_sync')
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
# actual table registration
args = [self.set_name, tbl, tgargs, None, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
if dest_table != tbl:
args[4] = dest_table
q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)"
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls):
# use full names
tbl = skytools.fq_name(tbl)
dest_table = self.options.dest_table or tbl
dest_table = skytools.fq_name(dest_table)
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
dst_db.commit()
self.set_lock_timeout(dst_curs)
if dest_table == tbl:
desc = tbl
else:
desc = "%s(%s)" % (tbl, dest_table)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching', desc)
else:
src_dest_table = src_tbls[tbl]['dest_table']
if not skytools.exists_table(src_curs, src_dest_table):
# table not present on provider - nowhere to get the DDL from
self.log.warning(
'Table %s missing on provider, cannot create, skipping',
desc)
return
schema = skytools.fq_name_parts(dest_table)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, src_dest_table)
src_db.commit()
# create, using rename logic only when necessary
newname = None
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs,
create_flags,
log=self.log,
new_table_name=newname)
elif not tbl_exists and self.options.skip_non_existing:
self.log.warning('Table %s does not exist on local node, skipping',
desc)
return
tgargs = self.build_tgargs()
attrs = {}
if self.options.handler:
attrs['handler'] = self.build_handler(tbl, tgargs,
self.options.dest_table)
if self.options.find_copy_node:
attrs['copy_node'] = '?'
elif self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
# actual table registration
args = [self.set_name, tbl, tgargs, None, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
if dest_table != tbl:
args[4] = dest_table
q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)"
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
if self.options.no_merge:
tgargs.append('no_merge')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs['handler'] = hstr
p.add(tgargs)
if self.options.expect_sync:
tgargs.append('expect_sync')
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.copy_condition:
attrs['copy_condition'] = self.options.copy_condition
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
args = [self.set_name, tbl, tgargs]
if attrs:
args.append(skytools.db_urlencode(attrs))
q = "select * from londiste.local_add_table(%s)" %\
','.join(['%s']*len(args))
# actual table registration
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls):
# use full names
tbl = skytools.fq_name(tbl)
dest_table = self.options.dest_table or tbl
dest_table = skytools.fq_name(dest_table)
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
if dest_table == tbl:
desc = tbl
else:
desc = "%s(%s)" % (tbl, dest_table)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % desc)
else:
src_dest_table = src_tbls[tbl]['dest_table']
if not skytools.exists_table(src_curs, src_dest_table):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table %s missing on provider, cannot create, skipping' % desc)
return
schema = skytools.fq_name_parts(dest_table)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, src_dest_table)
src_db.commit()
# create, using rename logic only when necessary
newname = None
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs, create_flags, log = self.log, new_table_name = newname)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
if self.options.no_merge:
tgargs.append('no_merge')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table)
attrs['handler'] = hstr
p.add(tgargs)
if self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
if self.options.expect_sync:
tgargs.append('expect_sync')
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
# actual table registration
args = [self.set_name, tbl, tgargs, None, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
if dest_table != tbl:
args[4] = dest_table
q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)"
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def match(self, objname, curs, tables, seqs):
return skytools.exists_schema(curs, objname)
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info("Table %s already exist, not touching" % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log=self.log)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append("tgflags=" + tgflags)
if self.options.no_triggers:
tgargs.append("no_triggers")
if self.options.merge_all:
tgargs.append("merge_all")
if self.options.no_merge:
tgargs.append("no_merge")
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs["handler"] = hstr
p.add(tgargs)
if self.options.expect_sync:
tgargs.append("expect_sync")
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs["skip_truncate"] = 1
if self.options.copy_condition:
attrs["copy_condition"] = self.options.copy_condition
if self.options.max_parallel_copy:
attrs["max_parallel_copy"] = self.options.max_parallel_copy
args = [self.set_name, tbl, tgargs]
if attrs:
args.append(skytools.db_urlencode(attrs))
q = "select * from londiste.local_add_table(%s)" % ",".join(["%s"] * len(args))
# actual table registration
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def match(self, objname, curs, tables, seqs):
return skytools.exists_schema(curs, objname)
