def save_stats (self):
"""
Dump client stats to database. Scheduled to be called periodically.
"""
# do not send stats if stats is missing or stats queue is missing
if not self.client_stats or not self.stat_queue_name:
return
now = time.time()
time_passed = now - self.last_stat_dump
self.log.info ("Sending usage stats to repository [%i]", len(self.client_stats))
# post role usage
usage = []
for client in self.client_stats.values():
self.log.trace ("client: %s", client)
usage.append (client.to_dict())
params = skytools.db_urlencode(dict(
hostname = self.hostname,
sample_length = '%d seconds' % time_passed,
snap_time = datetime.datetime.now().isoformat()))
confdb_funcargs = ('username=discovery', params, skytools.make_record_array(usage))
funcargs = [None, self.stat_queue_name, 'dba.set_role_usage',
skytools.db_urlencode(dict(enumerate(confdb_funcargs)))]
msg = DatabaseMessage (function = 'pgq.insert_event', params = cc.json.dumps(funcargs))
if self.msg_suffix:
msg.req += '.' + self.msg_suffix
self.main.ccpublish(msg)
self.client_stats = {}
self.last_stat_dump = now
def _make_record_convert(row):
"""Converts complex values."""
d = row.copy()
for k, v in d.items():
if isinstance(v, list):
d[k] = skytools.make_pgarray(v)
return skytools.db_urlencode(d)
def create_handler_string(name, arglist):
handler = name
if arglist:
args = _parse_arglist(arglist)
astr = skytools.db_urlencode(args)
handler = '%s(%s)' % (handler, astr)
return handler
def _make_record_convert(row):
"""Converts complex values."""
d = row.copy()
for k, v in d.items():
if isinstance(v, list):
d[k] = skytools.make_pgarray(v)
return skytools.db_urlencode(d)
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
# actual table registration
q = "select * from londiste.local_add_table(%s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
if self.options.expect_sync:
q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
if self.options.copy_condition:
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
attrs = {'copy_condition': self.options.copy_condition}
enc_attrs = skytools.db_urlencode(attrs)
self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs])
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs['handler'] = hstr
p.add(tgargs)
# actual table registration
q = "select * from londiste.local_add_table(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs])
if self.options.expect_sync:
q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
else:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.copy_condition:
attrs['copy_condition'] = self.options.copy_condition
if attrs:
enc_attrs = skytools.db_urlencode(attrs)
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs])
dst_db.commit()
def make_record(row):
""" Takes record as dict and returns it as urlencoded string.
Used to send data out of db service layer.or to fake incoming calls
"""
for v in row.values():
if isinstance(v, list):
return _make_record_convert(row)
return skytools.db_urlencode(row)
def make_record(row):
""" Takes record as dict and returns it as urlencoded string.
Used to send data out of db service layer.or to fake incoming calls
"""
for v in row.values():
if isinstance(v, list):
return _make_record_convert(row)
return skytools.db_urlencode(row)
def create_handler_string(name, arglist):
handler = name
if name.find('(') >= 0:
raise Exception('invalid handler name: %s' % name)
if arglist:
args = _parse_arglist(arglist)
astr = skytools.db_urlencode(args)
handler = '%s(%s)' % (handler, astr)
return handler
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
elif not tbl_exists:
self.log.warning('Table "%s" missing on subscriber, use --create if necessary' % tbl)
return
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs['handler'] = hstr
p.add(tgargs)
# actual table registration
q = "select * from londiste.local_add_table(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, tgargs])
if self.options.expect_sync:
q = "select * from londiste.local_set_table_state(%s, %s, NULL, 'ok')"
self.exec_cmd(dst_curs, q, [self.set_name, tbl])
else:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.copy_condition:
attrs['copy_condition'] = self.options.copy_condition
if attrs:
enc_attrs = skytools.db_urlencode(attrs)
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
self.exec_cmd(dst_curs, q, [self.set_name, tbl, enc_attrs])
dst_db.commit()
def create_handler_string(name, arglist):
handler = name
if name.find('(') >= 0:
raise Exception('invalid handler name: %s' % name)
if arglist:
args = _parse_arglist(arglist)
astr = skytools.db_urlencode(args)
handler = '%s(%s)' % (handler, astr)
return handler
def __init__(self, table_name, args, log):
"""Init per-batch table data cache."""
conf = args.copy()
# remove Multimaster args from conf
for name in ['func_name','func_conf']:
if name in conf:
conf.pop(name)
conf = skytools.db_urlencode(conf)
args = update(args, {'func_name': 'merge_on_time', 'func_conf': conf})
ApplyFuncHandler.__init__(self, table_name, args, log)
def __init__(self, table_name, args, dest_table):
"""Init per-batch table data cache."""
conf = args.copy()
# remove Multimaster args from conf
for name in ['func_name', 'func_conf']:
if name in conf:
conf.pop(name)
conf = skytools.db_urlencode(conf)
args = update(args, {'func_name': 'merge_on_time', 'func_conf': conf})
ApplyFuncHandler.__init__(self, table_name, args, dest_table)
def save_stats(self):
"""
Dump client stats to database. Scheduled to be called periodically.
"""
# do not send stats if stats is missing or stats queue is missing
if not self.client_stats or not self.stat_queue_name:
return
now = time.time()
time_passed = now - self.last_stat_dump
self.log.info("Sending usage stats to repository [%i]",
len(self.client_stats))
# post role usage
usage = []
for client in self.client_stats.values():
self.log.trace("client: %s", client)
usage.append(client.to_dict())
params = skytools.db_urlencode(
dict(hostname=self.hostname,
sample_length='%d seconds' % time_passed,
snap_time=datetime.datetime.now().isoformat()))
confdb_funcargs = ('username=discovery', params,
skytools.make_record_array(usage))
funcargs = [
None, self.stat_queue_name, 'dba.set_role_usage',
skytools.db_urlencode(dict(enumerate(confdb_funcargs)))
]
msg = DatabaseMessage(function='pgq.insert_event',
params=cc.json.dumps(funcargs))
if self.msg_suffix:
msg.req += '.' + self.msg_suffix
self.main.ccpublish(msg)
self.client_stats = {}
self.last_stat_dump = now
def cmd_resync(self, *args):
"""Reload data from provider node."""
db = self.get_database('db')
args = self.expand_arg_list(db, 'r', True, args)
if not self.options.find_copy_node:
self.load_local_info()
src_db = self.get_provider_db()
src_curs = src_db.cursor()
src_tbls = self.fetch_set_tables(src_curs)
src_db.commit()
problems = 0
for tbl in args:
tbl = skytools.fq_name(tbl)
if tbl not in src_tbls or not src_tbls[tbl]['local']:
self.log.error(
"Table %s does not exist on provider, need to switch to different provider",
tbl)
problems += 1
if problems > 0:
self.log.error("Problems, cancelling operation")
sys.exit(1)
if self.options.find_copy_node or self.options.copy_node:
q = "select table_name, table_attrs from londiste.get_table_list(%s) where local"
cur = db.cursor()
cur.execute(q, [self.set_name])
for row in cur.fetchall():
if row['table_name'] not in args:
continue
attrs = skytools.db_urldecode(row['table_attrs'] or '')
if self.options.find_copy_node:
attrs['copy_node'] = '?'
elif self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
s_attrs = skytools.db_urlencode(attrs)
q = "select * from londiste.local_set_table_attrs(%s, %s, %s)"
self.exec_cmd(db, q,
[self.set_name, row['table_name'], s_attrs])
q = "select * from londiste.local_set_table_state(%s, %s, null, null)"
self.exec_cmd_many(db, q, [self.set_name], args)
def cmd_change_handler(self, tbl):
"""Change handler (table_attrs) of the replicated table."""
self.load_local_info()
tbl = skytools.fq_name(tbl)
db = self.get_database('db')
curs = db.cursor()
q = "select table_attrs, dest_table "\
" from londiste.get_table_list(%s) "\
" where table_name = %s and local"
curs.execute(q, [self.set_name, tbl])
if curs.rowcount == 0:
self.log.error("Table %s not found on this node", tbl)
sys.exit(1)
attrs, dest_table = curs.fetchone()
attrs = skytools.db_urldecode(attrs or '')
old_handler = attrs.get('handler')
tgargs = self.build_tgargs()
if self.options.handler:
new_handler = self.build_handler(tbl, tgargs, dest_table)
else:
new_handler = None
if old_handler == new_handler:
self.log.info(
"Handler is already set to desired value, nothing done")
sys.exit(0)
if new_handler:
attrs['handler'] = new_handler
elif 'handler' in attrs:
del attrs['handler']
args = [self.set_name, tbl, tgargs, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
q = "select * from londiste.local_change_handler(%s, %s, %s, %s)"
self.exec_cmd(curs, q, args)
db.commit()
def cmd_change_handler(self, tbl):
"""Change handler (table_attrs) of the replicated table."""
self.load_local_info()
tbl = skytools.fq_name(tbl)
db = self.get_database('db')
curs = db.cursor()
q = "select table_attrs, dest_table "\
" from londiste.get_table_list(%s) "\
" where table_name = %s and local"
curs.execute(q, [self.set_name, tbl])
if curs.rowcount == 0:
self.log.error("Table %s not found on this node", tbl)
sys.exit(1)
attrs, dest_table = curs.fetchone()
attrs = skytools.db_urldecode(attrs or '')
old_handler = attrs.get('handler')
tgargs = self.build_tgargs()
if self.options.handler:
new_handler = self.build_handler(tbl, tgargs, dest_table)
else:
new_handler = None
if old_handler == new_handler:
self.log.info("Handler is already set to desired value, nothing done")
sys.exit(0)
if new_handler:
attrs['handler'] = new_handler
elif 'handler' in attrs:
del attrs['handler']
args = [self.set_name, tbl, tgargs, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
q = "select * from londiste.local_change_handler(%s, %s, %s, %s)"
self.exec_cmd(curs, q, args)
db.commit()
def cmd_resync(self, *args):
"""Reload data from provider node."""
db = self.get_database('db')
args = self.expand_arg_list(db, 'r', True, args)
if not self.options.find_copy_node:
self.load_local_info()
src_db = self.get_provider_db()
src_curs = src_db.cursor()
src_tbls = self.fetch_set_tables(src_curs)
src_db.commit()
problems = 0
for tbl in args:
tbl = skytools.fq_name(tbl)
if tbl not in src_tbls or not src_tbls[tbl]['local']:
self.log.error("Table %s does not exist on provider, need to switch to different provider", tbl)
problems += 1
if problems > 0:
self.log.error("Problems, cancelling operation")
sys.exit(1)
if self.options.find_copy_node or self.options.copy_node:
q = "select table_name, table_attrs from londiste.get_table_list(%s) where local"
cur = db.cursor()
cur.execute(q, [self.set_name])
for row in cur.fetchall():
if row['table_name'] not in args:
continue
attrs = skytools.db_urldecode (row['table_attrs'] or '')
if self.options.find_copy_node:
attrs['copy_node'] = '?'
elif self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
attrs = skytools.db_urlencode (attrs)
q = "select * from londiste.local_set_table_attrs (%s, %s, %s)"
self.exec_cmd(db, q, [self.set_name, row['table_name'], attrs])
q = "select * from londiste.local_set_table_state(%s, %s, null, null)"
self.exec_cmd_many(db, q, [self.set_name], args)
def make_record(row):
""" Takes record as dict and returns it as urlencoded string.
Used to send data out of db service layer.or to fake incoming calls
"""
return skytools.db_urlencode(row)
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log = self.log)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
if self.options.no_merge:
tgargs.append('no_merge')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs['handler'] = hstr
p.add(tgargs)
if self.options.expect_sync:
tgargs.append('expect_sync')
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.copy_condition:
attrs['copy_condition'] = self.options.copy_condition
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
args = [self.set_name, tbl, tgargs]
if attrs:
args.append(skytools.db_urlencode(attrs))
q = "select * from londiste.local_add_table(%s)" %\
','.join(['%s']*len(args))
# actual table registration
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def create_node(self, node_type, node_name, node_location):
"""Generic node init."""
provider_loc = self.options.provider
if node_type not in ('root', 'branch', 'leaf'):
raise Exception('unknown node type')
# connect to database
db = self.get_database("new_node", connstr=node_location)
# check if code is installed
self.install_code(db)
# query current status
res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)",
[self.queue_name])
info = res[0]
if info['node_type'] is not None:
self.log.info("Node is already initialized as %s" %
info['node_type'])
return
self.log.info("Initializing node")
node_attrs = {}
worker_name = self.options.worker
if not worker_name:
raise Exception('--worker required')
combined_queue = self.options.merge
if combined_queue and node_type != 'leaf':
raise Exception('--merge can be used only for leafs')
if self.options.sync_watermark:
if node_type != 'branch':
raise UsageError(
'--sync-watermark can be used only for branch nodes')
node_attrs['sync_watermark'] = self.options.sync_watermark
# register member
if node_type == 'root':
global_watermark = None
combined_queue = None
provider_name = None
self.exec_cmd(
db,
"select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
self.exec_cmd(
db,
"select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)",
[
self.queue_name, node_type, node_name, worker_name,
provider_name, global_watermark, combined_queue
])
provider_db = None
else:
if not provider_loc:
raise Exception('Please specify --provider')
root_db = self.find_root_db(provider_loc)
queue_info = self.load_queue_info(root_db)
# check if member already exists
if queue_info.get_member(node_name) is not None:
self.log.error("Node '%s' already exists" % node_name)
sys.exit(1)
combined_set = None
provider_db = self.get_database('provider_db',
connstr=provider_loc)
q = "select node_type, node_name from pgq_node.get_node_info(%s)"
res = self.exec_query(provider_db, q, [self.queue_name])
row = res[0]
if not row['node_name']:
raise Exception("provider node not found")
provider_name = row['node_name']
# register member on root
self.exec_cmd(
root_db,
"select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
# lookup provider
provider = queue_info.get_member(provider_name)
if not provider:
self.log.error("Node %s does not exist" % provider_name)
sys.exit(1)
# register on provider
self.exec_cmd(
provider_db,
"select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
rows = self.exec_cmd(
provider_db,
"select * from pgq_node.register_subscriber(%s, %s, %s, null)",
[self.queue_name, node_name, worker_name])
global_watermark = rows[0]['global_watermark']
# initialize node itself
# insert members
self.exec_cmd(
db,
"select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
for m in queue_info.member_map.values():
self.exec_cmd(
db,
"select * from pgq_node.register_location(%s, %s, %s, %s)",
[self.queue_name, m.name, m.location, m.dead])
# real init
self.exec_cmd(
db,
"select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)",
[
self.queue_name, node_type, node_name, worker_name,
provider_name, global_watermark, combined_queue
])
self.extra_init(node_type, db, provider_db)
if node_attrs:
s_attrs = skytools.db_urlencode(node_attrs)
self.exec_cmd(db, "select * from pgq_node.set_node_attrs(%s, %s)",
[self.queue_name, s_attrs])
self.log.info("Done")
def to_urlenc(self):
"""Convert container to urlencoded string."""
sdict = {}
for k, v in self.attrs.items():
sdict[k] = ','.join(v)
return skytools.db_urlencode(sdict)
def to_urlenc(self):
"""Convert container to urlencoded string."""
sdict = {}
for k, v in self.attrs.items():
sdict[k] = ','.join(v)
return skytools.db_urlencode(sdict)
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls):
# use full names
tbl = skytools.fq_name(tbl)
dest_table = self.options.dest_table or tbl
dest_table = skytools.fq_name(dest_table)
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
if dest_table == tbl:
desc = tbl
else:
desc = "%s(%s)" % (tbl, dest_table)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % desc)
else:
src_dest_table = src_tbls[tbl]['dest_table']
if not skytools.exists_table(src_curs, src_dest_table):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table %s missing on provider, cannot create, skipping' % desc)
return
schema = skytools.fq_name_parts(dest_table)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, src_dest_table)
src_db.commit()
# create, using rename logic only when necessary
newname = None
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs, create_flags, log = self.log, new_table_name = newname)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
if self.options.no_merge:
tgargs.append('no_merge')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table)
attrs['handler'] = hstr
p.add(tgargs)
if self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
if self.options.expect_sync:
tgargs.append('expect_sync')
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
# actual table registration
args = [self.set_name, tbl, tgargs, None, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
if dest_table != tbl:
args[4] = dest_table
q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)"
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls):
# use full names
tbl = skytools.fq_name(tbl)
dest_table = self.options.dest_table or tbl
dest_table = skytools.fq_name(dest_table)
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
if dest_table == tbl:
desc = tbl
else:
desc = "%s(%s)" % (tbl, dest_table)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching' % desc)
else:
src_dest_table = src_tbls[tbl]['dest_table']
if not skytools.exists_table(src_curs, src_dest_table):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table %s missing on provider, cannot create, skipping' % desc)
return
schema = skytools.fq_name_parts(dest_table)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, src_dest_table)
src_db.commit()
# create, using rename logic only when necessary
newname = None
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs, create_flags, log = self.log, new_table_name = newname)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append('tgflags='+tgflags)
if self.options.no_triggers:
tgargs.append('no_triggers')
if self.options.merge_all:
tgargs.append('merge_all')
if self.options.no_merge:
tgargs.append('no_merge')
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(
self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.options.dest_table)
attrs['handler'] = hstr
p.add(tgargs)
if self.options.find_copy_node:
attrs['copy_node'] = '?'
elif self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
if self.options.expect_sync:
tgargs.append('expect_sync')
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
# actual table registration
args = [self.set_name, tbl, tgargs, None, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
if dest_table != tbl:
args[4] = dest_table
q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)"
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags, src_tbls):
# use full names
tbl = skytools.fq_name(tbl)
dest_table = self.options.dest_table or tbl
dest_table = skytools.fq_name(dest_table)
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, dest_table)
dst_db.commit()
self.set_lock_timeout(dst_curs)
if dest_table == tbl:
desc = tbl
else:
desc = "%s(%s)" % (tbl, dest_table)
if create_flags:
if tbl_exists:
self.log.info('Table %s already exist, not touching', desc)
else:
src_dest_table = src_tbls[tbl]['dest_table']
if not skytools.exists_table(src_curs, src_dest_table):
# table not present on provider - nowhere to get the DDL from
self.log.warning(
'Table %s missing on provider, cannot create, skipping',
desc)
return
schema = skytools.fq_name_parts(dest_table)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, src_dest_table)
src_db.commit()
# create, using rename logic only when necessary
newname = None
if src_dest_table != dest_table:
newname = dest_table
s.create(dst_curs,
create_flags,
log=self.log,
new_table_name=newname)
elif not tbl_exists and self.options.skip_non_existing:
self.log.warning('Table %s does not exist on local node, skipping',
desc)
return
tgargs = self.build_tgargs()
attrs = {}
if self.options.handler:
attrs['handler'] = self.build_handler(tbl, tgargs,
self.options.dest_table)
if self.options.find_copy_node:
attrs['copy_node'] = '?'
elif self.options.copy_node:
attrs['copy_node'] = self.options.copy_node
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs['skip_truncate'] = 1
if self.options.max_parallel_copy:
attrs['max_parallel_copy'] = self.options.max_parallel_copy
# actual table registration
args = [self.set_name, tbl, tgargs, None, None]
if attrs:
args[3] = skytools.db_urlencode(attrs)
if dest_table != tbl:
args[4] = dest_table
q = "select * from londiste.local_add_table(%s, %s, %s, %s, %s)"
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def add_table(self, src_db, dst_db, tbl, create_flags):
src_curs = src_db.cursor()
dst_curs = dst_db.cursor()
tbl_exists = skytools.exists_table(dst_curs, tbl)
if create_flags:
if tbl_exists:
self.log.info("Table %s already exist, not touching" % tbl)
else:
if not skytools.exists_table(src_curs, tbl):
# table not present on provider - nowhere to get the DDL from
self.log.warning('Table "%s" missing on provider, skipping' % tbl)
return
schema = skytools.fq_name_parts(tbl)[0]
if not skytools.exists_schema(dst_curs, schema):
q = "create schema %s" % skytools.quote_ident(schema)
dst_curs.execute(q)
s = skytools.TableStruct(src_curs, tbl)
src_db.commit()
s.create(dst_curs, create_flags, log=self.log)
tgargs = []
if self.options.trigger_arg:
tgargs = self.options.trigger_arg
tgflags = self.options.trigger_flags
if tgflags:
tgargs.append("tgflags=" + tgflags)
if self.options.no_triggers:
tgargs.append("no_triggers")
if self.options.merge_all:
tgargs.append("merge_all")
if self.options.no_merge:
tgargs.append("no_merge")
attrs = {}
if self.options.handler:
hstr = londiste.handler.create_handler_string(self.options.handler, self.options.handler_arg)
p = londiste.handler.build_handler(tbl, hstr, self.log)
attrs["handler"] = hstr
p.add(tgargs)
if self.options.expect_sync:
tgargs.append("expect_sync")
if not self.options.expect_sync:
if self.options.skip_truncate:
attrs["skip_truncate"] = 1
if self.options.copy_condition:
attrs["copy_condition"] = self.options.copy_condition
if self.options.max_parallel_copy:
attrs["max_parallel_copy"] = self.options.max_parallel_copy
args = [self.set_name, tbl, tgargs]
if attrs:
args.append(skytools.db_urlencode(attrs))
q = "select * from londiste.local_add_table(%s)" % ",".join(["%s"] * len(args))
# actual table registration
self.exec_cmd(dst_curs, q, args)
dst_db.commit()
def create_node(self, node_type, node_name, node_location):
"""Generic node init."""
provider_loc = self.options.provider
if node_type not in ('root', 'branch', 'leaf'):
raise Exception('unknown node type')
# connect to database
db = self.get_database("new_node", connstr = node_location)
# check if code is installed
self.install_code(db)
# query current status
res = self.exec_query(db, "select * from pgq_node.get_node_info(%s)", [self.queue_name])
info = res[0]
if info['node_type'] is not None:
self.log.info("Node is already initialized as %s" % info['node_type'])
return
self.log.info("Initializing node")
node_attrs = {}
worker_name = self.options.worker
if not worker_name:
raise Exception('--worker required')
combined_queue = self.options.merge
if combined_queue and node_type != 'leaf':
raise Exception('--merge can be used only for leafs')
if self.options.sync_watermark:
if node_type != 'branch':
raise UsageError('--sync-watermark can be used only for branch nodes')
node_attrs['sync_watermark'] = self.options.sync_watermark
# register member
if node_type == 'root':
global_watermark = None
combined_queue = None
provider_name = None
self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
self.exec_cmd(db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)",
[self.queue_name, node_type, node_name, worker_name, provider_name, global_watermark, combined_queue])
provider_db = None
else:
if not provider_loc:
raise Exception('Please specify --provider')
root_db = self.find_root_db(provider_loc)
queue_info = self.load_queue_info(root_db)
# check if member already exists
if queue_info.get_member(node_name) is not None:
self.log.error("Node '%s' already exists" % node_name)
sys.exit(1)
combined_set = None
provider_db = self.get_database('provider_db', connstr = provider_loc)
q = "select node_type, node_name from pgq_node.get_node_info(%s)"
res = self.exec_query(provider_db, q, [self.queue_name])
row = res[0]
if not row['node_name']:
raise Exception("provider node not found")
provider_name = row['node_name']
# register member on root
self.exec_cmd(root_db, "select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
# lookup provider
provider = queue_info.get_member(provider_name)
if not provider:
self.log.error("Node %s does not exist" % provider_name)
sys.exit(1)
# register on provider
self.exec_cmd(provider_db, "select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
rows = self.exec_cmd(provider_db, "select * from pgq_node.register_subscriber(%s, %s, %s, null)",
[self.queue_name, node_name, worker_name])
global_watermark = rows[0]['global_watermark']
# initialize node itself
# insert members
self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, false)",
[self.queue_name, node_name, node_location])
for m in queue_info.member_map.values():
self.exec_cmd(db, "select * from pgq_node.register_location(%s, %s, %s, %s)",
[self.queue_name, m.name, m.location, m.dead])
# real init
self.exec_cmd(db, "select * from pgq_node.create_node(%s, %s, %s, %s, %s, %s, %s)",
[ self.queue_name, node_type, node_name, worker_name,
provider_name, global_watermark, combined_queue ])
self.extra_init(node_type, db, provider_db)
if node_attrs:
s_attrs = skytools.db_urlencode(node_attrs)
self.exec_cmd(db, "select * from pgq_node.set_node_attrs(%s, %s)",
[self.queue_name, s_attrs])
self.log.info("Done")
