def work(self):
"""Syncer main function."""
# 'SELECT 1' and COPY must use same snapshot, so change isolation level.
dst_db = self.get_database('db', isolation_level = skytools.I_REPEATABLE_READ)
pnode, ploc = self.get_provider_location(dst_db)
dst_tables, names = self.get_tables(dst_db)
if len(self.args) > 2:
tlist = self.args[2:]
else:
tlist = names
for tbl in tlist:
tbl = skytools.fq_name(tbl)
if not tbl in dst_tables:
self.log.warning('Table not subscribed: %s' % tbl)
continue
t2 = dst_tables[tbl]
if t2.merge_state != 'ok':
self.log.warning('Table %s not synced yet, no point' % tbl)
continue
pnode, ploc, wname = find_copy_source(self, self.queue_name, tbl, pnode, ploc)
self.log.info('%s: Using node %s as provider', tbl, pnode)
if wname is None:
wname = self.consumer_name
self.downstream_worker_name = wname
self.process_one_table(tbl, t2, dst_db, pnode, ploc)
# signal caller about bad tables
sys.exit(self.bad_tables)
def register_copy_consumer(self):
dst_db = self.get_database('db')
dst_curs = dst_db.cursor()
# fetch table attrs
q = "select * from londiste.get_table_list(%s) where table_name = %s"
dst_curs.execute(q, [self.queue_name, self.copy_table_name])
rows = dst_curs.fetchall()
attrs = {}
if len(rows) > 0:
v_attrs = rows[0]['table_attrs']
if v_attrs:
attrs = skytools.db_urldecode(v_attrs)
# fetch parent consumer state
q = "select * from pgq_node.get_consumer_state(%s, %s)"
rows = self.exec_cmd(dst_db, q,
[self.queue_name, self.old_consumer_name])
state = rows[0]
source_node = state['provider_node']
source_location = state['provider_location']
# do we have node here?
if 'copy_node' in attrs:
if attrs['copy_node'] == '?':
source_node, source_location, wname = find_copy_source(
self, self.queue_name, self.copy_table_name, source_node,
source_location)
else:
# take node from attrs
source_node = attrs['copy_node']
q = "select * from pgq_node.get_queue_locations(%s) where node_name = %s"
dst_curs.execute(q, [self.queue_name, source_node])
rows = dst_curs.fetchall()
if len(rows):
source_location = rows[0]['node_location']
self.log.info("Using '%s' as source node", source_node)
self.register_consumer(source_location)
def register_copy_consumer(self):
dst_db = self.get_database('db')
dst_curs = dst_db.cursor()
# fetch table attrs
q = "select * from londiste.get_table_list(%s) where table_name = %s"
dst_curs.execute(q, [self.queue_name, self.copy_table_name])
rows = dst_curs.fetchall()
attrs = {}
if len(rows) > 0:
v_attrs = rows[0]['table_attrs']
if v_attrs:
attrs = skytools.db_urldecode(v_attrs)
# fetch parent consumer state
q = "select * from pgq_node.get_consumer_state(%s, %s)"
rows = self.exec_cmd(dst_db, q, [self.queue_name, self.old_consumer_name])
state = rows[0]
source_node = state['provider_node']
source_location = state['provider_location']
# do we have node here?
if 'copy_node' in attrs:
if attrs['copy_node'] == '?':
source_node, source_location, ___wname = find_copy_source(self,
self.queue_name, self.copy_table_name, source_node, source_location)
else:
# take node from attrs
source_node = attrs['copy_node']
q = "select * from pgq_node.get_queue_locations(%s) where node_name = %s"
dst_curs.execute(q, [self.queue_name, source_node])
rows = dst_curs.fetchall()
if len(rows):
source_location = rows[0]['node_location']
self.log.info("Using '%s' as source node", source_node)
self.register_consumer(source_location)
def work(self):
"""Syncer main function."""
# 'SELECT 1' and COPY must use same snapshot, so change isolation level.
dst_db = self.get_database('db',
isolation_level=skytools.I_REPEATABLE_READ)
pnode, ploc = self.get_provider_location(dst_db)
dst_tables, names = self.get_tables(dst_db)
if len(self.args) > 2:
tlist = self.args[2:]
else:
tlist = names
for tbl in tlist:
tbl = skytools.fq_name(tbl)
if not tbl in dst_tables:
self.log.warning('Table not subscribed: %s', tbl)
continue
t2 = dst_tables[tbl]
if t2.merge_state != 'ok':
self.log.warning('Table %s not synced yet, no point', tbl)
continue
pnode, ploc, wname = find_copy_source(self, self.queue_name, tbl,
pnode, ploc)
self.log.info('%s: Using node %s as provider', tbl, pnode)
if wname is None:
wname = self.consumer_name
self.downstream_worker_name = wname
self.process_one_table(tbl, t2, dst_db, pnode, ploc)
# signal caller about bad tables
sys.exit(self.bad_tables)
def cmd_add_table(self, *args):
"""Attach table(s) to local node."""
self.load_local_info()
src_db = self.get_provider_db()
if not self.is_root():
src_curs = src_db.cursor()
src_tbls = self.fetch_set_tables(src_curs)
src_db.commit()
dst_db = self.get_database('db')
dst_curs = dst_db.cursor()
dst_tbls = self.fetch_set_tables(dst_curs)
if self.is_root():
src_tbls = dst_tbls
else:
self.sync_table_list(dst_curs, src_tbls, dst_tbls)
dst_db.commit()
needs_tbl = self.handler_needs_table()
args = self.expand_arg_list(dst_db, 'r', False, args, needs_tbl)
# pick proper create flags
if self.options.create_full:
create_flags = skytools.T_ALL
elif self.options.create:
create_flags = skytools.T_TABLE | skytools.T_PKEY
else:
create_flags = 0
# search for usable copy node if requested & needed
if (self.options.find_copy_node and create_flags != 0
and needs_tbl and not self.is_root()):
src_name, src_loc, _ = find_copy_source(self, self.queue_name, args, None, self.provider_location)
self.options.copy_node = src_name
self.close_database('provider_db')
src_db = self.get_provider_db()
src_curs = src_db.cursor()
src_tbls = self.fetch_set_tables(src_curs)
src_db.commit()
# dont check for exist/not here (root handling)
if not self.is_root() and not self.options.expect_sync and not self.options.find_copy_node:
problems = False
for tbl in args:
tbl = skytools.fq_name(tbl)
if (tbl in src_tbls) and not src_tbls[tbl]['local']:
if self.options.skip_non_existing:
self.log.warning("Table %s does not exist on provider", tbl)
else:
self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl)
problems = True
if problems:
self.log.error("Problems, canceling operation")
sys.exit(1)
# sanity check
if self.options.dest_table and len(args) > 1:
self.log.error("--dest-table can be given only for single table")
sys.exit(1)
# seems ok
for tbl in args:
self.add_table(src_db, dst_db, tbl, create_flags, src_tbls)
# wait
if self.options.wait_sync:
self.wait_for_sync(dst_db)
def cmd_add_table(self, *args):
"""Attach table(s) to local node."""
self.load_local_info()
src_db = self.get_provider_db()
if not self.is_root():
src_curs = src_db.cursor()
src_tbls = self.fetch_set_tables(src_curs)
src_db.commit()
dst_db = self.get_database('db')
dst_curs = dst_db.cursor()
dst_tbls = self.fetch_set_tables(dst_curs)
if self.is_root():
src_tbls = dst_tbls
else:
self.sync_table_list(dst_curs, src_tbls, dst_tbls)
dst_db.commit()
needs_tbl = self.handler_needs_table()
args = self.expand_arg_list(dst_db, 'r', False, args, needs_tbl)
# pick proper create flags
if self.options.create_full:
create_flags = skytools.T_ALL
elif self.options.create:
create_flags = skytools.T_TABLE | skytools.T_PKEY
else:
create_flags = 0
# search for usable copy node if requested & needed
if (self.options.find_copy_node and create_flags != 0 and needs_tbl
and not self.is_root()):
src_name, _, _ = find_copy_source(self, self.queue_name, args,
None, self.provider_location)
self.options.copy_node = src_name
self.close_database('provider_db')
src_db = self.get_provider_db()
src_curs = src_db.cursor()
src_tbls = self.fetch_set_tables(src_curs)
src_db.commit()
# dont check for exist/not here (root handling)
if not self.is_root(
) and not self.options.expect_sync and not self.options.find_copy_node:
problems = False
for tbl in args:
tbl = skytools.fq_name(tbl)
if (tbl in src_tbls) and not src_tbls[tbl]['local']:
if self.options.skip_non_existing:
self.log.warning("Table %s does not exist on provider",
tbl)
else:
self.log.error(
"Table %s does not exist on provider, need to switch to different provider",
tbl)
problems = True
if problems:
self.log.error("Problems, canceling operation")
sys.exit(1)
# sanity check
if self.options.dest_table and len(args) > 1:
self.log.error("--dest-table can be given only for single table")
sys.exit(1)
# seems ok
for tbl in args:
self.add_table(src_db, dst_db, tbl, create_flags, src_tbls)
# wait
if self.options.wait_sync:
self.wait_for_sync(dst_db)