def change_config(self):
if len(self.args) < 3:
list = self.get_queue_list()
for qname in list:
self.show_config(qname)
return
qname = self.args[2]
if len(self.args) == 3:
self.show_config(qname)
return
alist = []
for el in self.args[3:]:
k, v = el.split('=')
if k not in config_allowed_list:
qk = "queue_" + k
if qk not in config_allowed_list:
raise Exception('unknown config var: '+k)
k = qk
expr = "%s=%s" % (k, skytools.quote_literal(v))
alist.append(expr)
self.log.info('Change queue %s config to: %s' % (qname, ", ".join(alist)))
sql = "update pgq.queue set %s where queue_name = %s" % (
", ".join(alist), skytools.quote_literal(qname))
self.exec_sql(sql, [])
def mk_update_sql(row, tbl, pkey_list, field_map = None):
r"""Generate UPDATE statement from dict data.
>>> mk_update_sql({'id': 0, 'id2': '2', 'data': 'str\\'}, 'Table', ['id', 'id2'])
'update only public."Table" set data = E\'str\\\\\' where id = \'0\' and id2 = \'2\';'
"""
if len(pkey_list) < 1:
raise Exception("update needs pkeys")
set_list = []
whe_list = []
pkmap = {}
for k in pkey_list:
pkmap[k] = 1
new_k = field_map and field_map[k] or k
col = skytools.quote_ident(new_k)
val = skytools.quote_literal(row[k])
whe_list.append("%s = %s" % (col, val))
if field_map:
for src, dst in field_map.iteritems():
if src not in pkmap:
col = skytools.quote_ident(dst)
val = skytools.quote_literal(row[src])
set_list.append("%s = %s" % (col, val))
else:
for col, val in row.iteritems():
if col not in pkmap:
col = skytools.quote_ident(col)
val = skytools.quote_literal(val)
set_list.append("%s = %s" % (col, val))
return "update only %s set %s where %s;" % (skytools.quote_fqident(tbl),
", ".join(set_list), " and ".join(whe_list))
def change_config(self):
if len(self.args) < 3:
qlist = self.get_queue_list()
for qname in qlist:
self.show_config(qname)
return
qname = self.args[2]
if len(self.args) == 3:
self.show_config(qname)
return
alist = []
for el in self.args[3:]:
k, v = el.split('=')
if k not in config_allowed_list:
qk = "queue_" + k
if qk not in config_allowed_list:
raise Exception('unknown config var: ' + k)
k = qk
expr = "%s=%s" % (k, skytools.quote_literal(v))
alist.append(expr)
self.log.info('Change queue %s config to: %s' %
(qname, ", ".join(alist)))
sql = "update pgq.queue set %s where queue_name = %s" % (
", ".join(alist), skytools.quote_literal(qname))
self.exec_sql(sql, [])
def mk_insert_sql(row, tbl, pkey_list=None, field_map=None):
"""Generate INSERT statement from dict data.
>>> from collections import OrderedDict
>>> row = OrderedDict([('id',1), ('data', None)])
>>> mk_insert_sql(row, 'tbl')
"insert into public.tbl (id, data) values ('1', null);"
>>> mk_insert_sql(row, 'tbl', ['x'], OrderedDict([('id', 'id_'), ('data', 'data_')]))
"insert into public.tbl (id_, data_) values ('1', null);"
"""
col_list = []
val_list = []
if field_map:
for src, dst in field_map.items():
col_list.append(skytools.quote_ident(dst))
val_list.append(skytools.quote_literal(row[src]))
else:
for c, v in row.items():
col_list.append(skytools.quote_ident(c))
val_list.append(skytools.quote_literal(v))
col_str = ", ".join(col_list)
val_str = ", ".join(val_list)
return "insert into %s (%s) values (%s);" % (
skytools.quote_fqident(tbl), col_str, val_str)
def _new_obj_sql(self, queue, objname, objkind):
args = {
"queue": skytools.quote_literal(queue),
"obj": objname,
"ifield": objname + "_name",
"itable": "londiste." + objname + "_info",
"kind": skytools.quote_literal(objkind),
}
q = (
"""select quote_ident(n.nspname) || '.' || quote_ident(r.relname)
from pg_catalog.pg_class r
join pg_catalog.pg_namespace n on (n.oid = r.relnamespace)
left join %(itable)s i
on (i.queue_name = %(queue)s and
i.%(ifield)s = (n.nspname || '.' || r.relname))
where r.relkind = %(kind)s
and n.nspname not in ('pg_catalog', 'information_schema', 'pgq', 'londiste', 'pgq_node', 'pgq_ext')
and n.nspname !~ 'pg_.*'
and i.%(ifield)s is null
union all
select londiste.quote_fqname(%(ifield)s) from %(itable)s
where queue_name = %(queue)s and not local
order by 1 """
% args
)
return q
def mk_update_sql(row, tbl, pkey_list, field_map=None):
r"""Generate UPDATE statement from dict data.
>>> mk_update_sql({'id': 0, 'id2': '2', 'data': 'str\\'}, 'Table', ['id', 'id2'])
'update only public."Table" set data = E\'str\\\\\' where id = \'0\' and id2 = \'2\';'
"""
if len(pkey_list) < 1:
raise Exception("update needs pkeys")
set_list = []
whe_list = []
pkmap = {}
for k in pkey_list:
pkmap[k] = 1
new_k = field_map and field_map[k] or k
col = skytools.quote_ident(new_k)
val = skytools.quote_literal(row[k])
whe_list.append("%s = %s" % (col, val))
if field_map:
for src, dst in field_map.iteritems():
if src not in pkmap:
col = skytools.quote_ident(dst)
val = skytools.quote_literal(row[src])
set_list.append("%s = %s" % (col, val))
else:
for col, val in row.iteritems():
if col not in pkmap:
col = skytools.quote_ident(col)
val = skytools.quote_literal(val)
set_list.append("%s = %s" % (col, val))
return "update only %s set %s where %s;" % (skytools.quote_fqident(tbl),
", ".join(set_list),
" and ".join(whe_list))
def mk_insert_sql(row, tbl, pkey_list=None, field_map=None):
"""Generate INSERT statement from dict data.
>>> from collections import OrderedDict
>>> row = OrderedDict([('id',1), ('data', None)])
>>> mk_insert_sql(row, 'tbl')
"insert into public.tbl (id, data) values ('1', null);"
>>> mk_insert_sql(row, 'tbl', ['x'], OrderedDict([('id', 'id_'), ('data', 'data_')]))
"insert into public.tbl (id_, data_) values ('1', null);"
"""
col_list = []
val_list = []
if field_map:
for src, dst in field_map.items():
col_list.append(skytools.quote_ident(dst))
val_list.append(skytools.quote_literal(row[src]))
else:
for c, v in row.items():
col_list.append(skytools.quote_ident(c))
val_list.append(skytools.quote_literal(v))
col_str = ", ".join(col_list)
val_str = ", ".join(val_list)
return "insert into %s (%s) values (%s);" % (skytools.quote_fqident(tbl),
col_str, val_str)
def __init__(self, curs, table_name):
"""Initializes class by loading info about table_name from database."""
super(TableStruct, self).__init__(curs, table_name)
self.table_name = table_name
# fill args
schema, name = skytools.fq_name_parts(table_name)
args = {
'schema': schema,
'table': name,
'fqname': self.fqname,
'fq2name': skytools.quote_literal(self.fqname),
'oid': skytools.get_table_oid(curs, table_name),
'pg_class_oid': skytools.get_table_oid(curs,
'pg_catalog.pg_class'),
}
# load table struct
self.col_list = self._load_elem(curs, self.name, args, TColumn)
# if db is GP then read also table distribution keys
if skytools.exists_table(curs, "pg_catalog.gp_distribution_policy"):
self.dist_key_list = self._load_elem(curs, self.name, args,
TGPDistKey)
else:
self.dist_key_list = None
self.object_list = [
TTable(table_name, self.col_list, self.dist_key_list)
]
self.seq_list = []
# load seqs
for col in self.col_list:
if col.seqname:
fqname = quote_fqident(col.seqname)
owner = self.fqname + '.' + quote_ident(col.name)
seq_args = {
'fqname': fqname,
'fq2name': skytools.quote_literal(fqname),
'owner': skytools.quote_literal(owner),
}
self.seq_list += self._load_elem(curs, col.seqname, seq_args,
TSeq)
self.object_list += self.seq_list
# load additional objects
to_load = [
TColumnDefault, TConstraint, TIndex, TTrigger, TRule, TGrant,
TOwner, TParent
]
for eclass in to_load:
self.object_list += self._load_elem(curs, self.name, args, eclass)
def get_batch_list(self):
if not self.cur_queue:
return []
qname = skytools.quote_literal(self.cur_queue)
q = "select current_batch::text from pgq.get_consumer_info(%s)"\
" where current_batch is not null order by 1" % qname
return self._ccache('batch_list', q, 'pgq')
def _gen_dict_insert(tbl, row, fields, qfields):
tmp = []
for f in fields:
v = row.get(f)
tmp.append(skytools.quote_literal(v))
fmt = "insert into %s (%s) values (%s);"
return fmt % (tbl, ",".join(qfields), ",".join(tmp))
def addcmp(self, list, f, v):
if v is None:
s = "%s is null" % f
else:
vq = skytools.quote_literal(v)
s = "%s = %s" % (f, vq)
list.append(s)
def _gen_dict_insert(tbl, row, fields, qfields):
tmp = []
for f in fields:
v = row.get(f)
tmp.append(skytools.quote_literal(v))
fmt = "insert into %s (%s) values (%s);"
return fmt % (tbl, ",".join(qfields), ",".join(tmp))
def get_batch_list(self):
if not self.cur_queue:
return []
qname = skytools.quote_literal(self.cur_queue)
q = "select current_batch::text from pgq.get_consumer_info(%s)"\
" where current_batch is not null order by 1" % qname
return self._ccache('batch_list', q, 'pgq')
def collect_meta(self, table_list, meta, args):
"""Display fkey/trigger info."""
if args == []:
args = ['pending', 'active']
field_map = {'triggers': ['table_name', 'trigger_name', 'trigger_def'],
'fkeys': ['from_table', 'to_table', 'fkey_name', 'fkey_def']}
query_map = {'pending': "select %s from londiste.subscriber_get_table_pending_%s(%%s)",
'active' : "select %s from londiste.find_table_%s(%%s)"}
table_list = self.clean_subscriber_tables(table_list)
if len(table_list) == 0:
self.log.info("No tables, no fkeys")
return
dst_db = self.get_database('subscriber_db')
dst_curs = dst_db.cursor()
for which in args:
union_list = []
fields = field_map[meta]
q = query_map[which] % (",".join(fields), meta)
for tbl in table_list:
union_list.append(q % skytools.quote_literal(tbl))
# use union as fkey may appear in duplicate
sql = " union ".join(union_list) + " order by 1"
desc = "%s %s" % (which, meta)
self.display_table(desc, dst_curs, fields, sql)
dst_db.commit()
def mk_delete_sql(self, tbl, key_list, data):
# generate delete command
whe_list = []
for k in key_list:
whe_list.append("%s = %s" % (skytools.quote_ident(k), skytools.quote_literal(data[k])))
whe_str = " and ".join(whe_list)
return "delete from %s where %s;" % (skytools.quote_fqident(tbl), whe_str)
def mk_delete_sql(row, tbl, pkey_list, field_map=None):
"""Generate DELETE statement from dict data.
>>> mk_delete_sql({'a': 1, 'b':2, 'c':3}, 'tablename', ['a','b'])
"delete from only public.tablename where a = '1' and b = '2';"
>>> mk_delete_sql({'a': 1, 'b':2, 'c':3}, 'tablename', ['a','b'], {'a': 'aa', 'b':'bb'})
"delete from only public.tablename where aa = '1' and bb = '2';"
>>> mk_delete_sql({'a': 1, 'b':2, 'c':3}, 'tablename', [])
Traceback (most recent call last):
...
Exception: delete needs pkeys
"""
if len(pkey_list) < 1:
raise Exception("delete needs pkeys")
whe_list = []
for k in pkey_list:
new_k = field_map and field_map[k] or k
col = skytools.quote_ident(new_k)
val = skytools.quote_literal(row[k])
whe_list.append("%s = %s" % (col, val))
whe_str = " and ".join(whe_list)
return "delete from only %s where %s;" % (skytools.quote_fqident(tbl),
whe_str)
def process_remote_batch(self, src_db, tick_id, ev_list, dst_db):
"All work for a batch. Entry point from SetConsumer."
# this part can play freely with transactions
if not self.code_check_done:
self.check_code(dst_db)
self.code_check_done = 1
self.sync_database_encodings(src_db, dst_db)
self.cur_tick = self.batch_info["tick_id"]
self.prev_tick = self.batch_info["prev_tick_id"]
dst_curs = dst_db.cursor()
self.load_table_state(dst_curs)
self.sync_tables(src_db, dst_db)
self.copy_snapshot_cleanup(dst_db)
# only main thread is allowed to restore fkeys
if not self.copy_thread:
self.restore_fkeys(dst_db)
for p in self.used_plugins.values():
p.reset()
self.used_plugins = {}
# now the actual event processing happens.
# they must be done all in one tx in dst side
# and the transaction must be kept open so that
# the cascade-consumer can save last tick and commit.
self.sql_list = []
CascadedWorker.process_remote_batch(self, src_db, tick_id, ev_list, dst_db)
self.flush_sql(dst_curs)
for p in self.used_plugins.values():
p.finish_batch(self.batch_info, dst_curs)
self.used_plugins = {}
# finalize table changes
self.save_table_state(dst_curs)
# store event filter
if self.cf.getboolean("local_only", False):
if self.copy_thread:
_filterlist = skytools.quote_literal(self.copy_table_name)
else:
_filterlist = ",".join(map(skytools.quote_literal, self.table_map.keys()))
self.consumer_filter = (
"""
((ev_type like 'pgq%%' or ev_type like 'londiste%%')
or (ev_extra1 in (%s)))
"""
% _filterlist
)
else:
self.consumer_filter = None
def get_known_seq_list(self):
if not self.cur_queue:
return []
qname = skytools.quote_literal(self.cur_queue)
q = "select londiste.quote_fqname(seq_name)"\
" from londiste.seq_info"\
" where queue_name = %s order by 1" % qname
return self._ccache('known_seq_list', q, 'londiste')
def addcmp(self, dst_list, f, v):
"""Add quoted comparison."""
if v is None:
s = "%s is null" % f
else:
vq = skytools.quote_literal(v)
s = "%s = %s" % (f, vq)
dst_list.append(s)
def addcmp(self, list, f, v):
"""Add quoted comparision."""
if v is None:
s = "%s is null" % f
else:
vq = skytools.quote_literal(v)
s = "%s = %s" % (f, vq)
list.append(s)
def process_remote_batch(self, src_db, tick_id, ev_list, dst_db):
"All work for a batch. Entry point from SetConsumer."
# this part can play freely with transactions
if not self.code_check_done:
self.check_code(dst_db)
self.code_check_done = 1
self.sync_database_encodings(src_db, dst_db)
self.cur_tick = self.batch_info['tick_id']
self.prev_tick = self.batch_info['prev_tick_id']
dst_curs = dst_db.cursor()
self.load_table_state(dst_curs)
self.sync_tables(src_db, dst_db)
self.copy_snapshot_cleanup(dst_db)
# only main thread is allowed to restore fkeys
if not self.copy_thread:
self.restore_fkeys(dst_db)
for p in self.used_plugins.values():
p.reset()
self.used_plugins = {}
# now the actual event processing happens.
# they must be done all in one tx in dst side
# and the transaction must be kept open so that
# the cascade-consumer can save last tick and commit.
self.sql_list = []
CascadedWorker.process_remote_batch(self, src_db, tick_id, ev_list,
dst_db)
self.flush_sql(dst_curs)
for p in self.used_plugins.values():
p.finish_batch(self.batch_info, dst_curs)
self.used_plugins = {}
# finalize table changes
self.save_table_state(dst_curs)
# store event filter
if self.cf.getboolean('local_only', False):
if self.copy_thread:
_filterlist = skytools.quote_literal(self.copy_table_name)
else:
_filterlist = ','.join(
map(skytools.quote_literal, self.table_map.keys()))
self.consumer_filter = """
((ev_type like 'pgq%%' or ev_type like 'londiste%%')
or (ev_extra1 in (%s)))
""" % _filterlist
else:
self.consumer_filter = None
def __str__(self):
if self.conf.param_type == PARAM_INLINE:
return skytools.quote_literal(self.value)
elif self.conf.param_type == PARAM_DBAPI:
return "%s"
elif self.conf.param_type == PARAM_PLPY:
return "$%d" % self.pos
else:
raise Exception("bad QArgConf.param_type")
def __str__(self):
if self.conf.param_type == PARAM_INLINE:
return skytools.quote_literal(self.value)
elif self.conf.param_type == PARAM_DBAPI:
return "%s"
elif self.conf.param_type == PARAM_PLPY:
return "$%d" % self.pos
else:
raise Exception("bad QArgConf.param_type")
def __init__(self, curs, table_name):
"""Initializes class by loading info about table_name from database."""
BaseStruct.__init__(self, curs, table_name)
self.table_name = table_name
# fill args
schema, name = skytools.fq_name_parts(table_name)
args = {
'schema': schema,
'table': name,
'fqname': self.fqname,
'fq2name': skytools.quote_literal(self.fqname),
'oid': skytools.get_table_oid(curs, table_name),
'pg_class_oid': skytools.get_table_oid(curs, 'pg_catalog.pg_class'),
}
# load table struct
self.col_list = self._load_elem(curs, self.name, args, TColumn)
# if db is GP then read also table distribution keys
if skytools.exists_table(curs, "pg_catalog.gp_distribution_policy"):
self.dist_key_list = self._load_elem(curs, self.name, args,
TGPDistKey)
else:
self.dist_key_list = None
self.object_list = [ TTable(table_name, self.col_list,
self.dist_key_list) ]
self.seq_list = []
# load seqs
for col in self.col_list:
if col.seqname:
fqname = quote_fqident(col.seqname)
owner = self.fqname + '.' + quote_ident(col.name)
seq_args = { 'fqname': fqname, 'owner': skytools.quote_literal(owner) }
self.seq_list += self._load_elem(curs, col.seqname, seq_args, TSeq)
self.object_list += self.seq_list
# load additional objects
to_load = [TColumnDefault, TConstraint, TIndex, TTrigger,
TRule, TGrant, TOwner, TParent]
for eclass in to_load:
self.object_list += self._load_elem(curs, self.name, args, eclass)
def _gen_list_insert(tbl, row, fields, qfields):
tmp = []
for i in range(len(fields)):
try:
v = row[i]
except IndexError:
v = None
tmp.append(skytools.quote_literal(v))
fmt = "insert into %s (%s) values (%s);"
return fmt % (tbl, ",".join(qfields), ",".join(tmp))
def get_known_seq_list(self):
if not self.cur_queue:
return []
qname = skytools.quote_literal(self.cur_queue)
q = (
"select londiste.quote_fqname(seq_name)"
" from londiste.seq_info"
" where queue_name = %s order by 1" % qname
)
return self._ccache("known_seq_list", q, "londiste")
def got_missed_insert(self, tbl, src_row):
self.cnt_insert += 1
fld_list = self.common_fields
val_list = []
for f in fld_list:
v = unescape(src_row[f])
val_list.append(skytools.quote_literal(v))
q = "insert into %s (%s) values (%s);" % (
tbl, ", ".join(fld_list), ", ".join(val_list))
self.show_fix(tbl, q, 'insert')
def mk_delete_sql(self, tbl, key_list, data):
# generate delete command
whe_list = []
for k in key_list:
whe_list.append(
"%s = %s" %
(skytools.quote_ident(k), skytools.quote_literal(data[k])))
whe_str = " and ".join(whe_list)
return "delete from %s where %s;" % (skytools.quote_fqident(tbl),
whe_str)
def _gen_list_insert(tbl, row, fields, qfields):
tmp = []
for i in range(len(fields)):
try:
v = row[i]
except IndexError:
v = None
tmp.append(skytools.quote_literal(v))
fmt = "insert into %s (%s) values (%s);"
return fmt % (tbl, ",".join(qfields), ",".join(tmp))
def mk_insert_sql(self, tbl, key_list, data):
# generate insert command
col_list = []
val_list = []
for c, v in data.items():
col_list.append(skytools.quote_ident(c))
val_list.append(skytools.quote_literal(v))
col_str = ",".join(col_list)
val_str = ",".join(val_list)
return "insert into %s (%s) values (%s);" % (
skytools.quote_fqident(tbl), col_str, val_str)
def mk_insert_sql(self, tbl, key_list, data):
# generate insert command
col_list = []
val_list = []
for c, v in data.items():
col_list.append(skytools.quote_ident(c))
val_list.append(skytools.quote_literal(v))
col_str = ",".join(col_list)
val_str = ",".join(val_list)
return "insert into %s (%s) values (%s);" % (
skytools.quote_fqident(tbl), col_str, val_str)
def mk_insert_sql(row, tbl, pkey_list=None, field_map=None):
"""Generate INSERT statement from dict data.
>>> mk_insert_sql({'id': '1', 'data': None}, 'tbl')
"insert into public.tbl (data, id) values (null, '1');"
"""
col_list = []
val_list = []
if field_map:
for src, dst in field_map.iteritems():
col_list.append(skytools.quote_ident(dst))
val_list.append(skytools.quote_literal(row[src]))
else:
for c, v in row.iteritems():
col_list.append(skytools.quote_ident(c))
val_list.append(skytools.quote_literal(v))
col_str = ", ".join(col_list)
val_str = ", ".join(val_list)
return "insert into %s (%s) values (%s);" % (skytools.quote_fqident(tbl), col_str, val_str)
def mk_insert_sql(row, tbl, pkey_list=None, field_map=None):
"""Generate INSERT statement from dict data.
>>> mk_insert_sql({'id': '1', 'data': None}, 'tbl')
"insert into public.tbl (data, id) values (null, '1');"
"""
col_list = []
val_list = []
if field_map:
for src, dst in field_map.iteritems():
col_list.append(skytools.quote_ident(dst))
val_list.append(skytools.quote_literal(row[src]))
else:
for c, v in row.iteritems():
col_list.append(skytools.quote_ident(c))
val_list.append(skytools.quote_literal(v))
col_str = ", ".join(col_list)
val_str = ", ".join(val_list)
return "insert into %s (%s) values (%s);" % (skytools.quote_fqident(tbl),
col_str, val_str)
def got_missed_insert(self, tbl, src_row):
"""Create sql for missed insert."""
self.cnt_insert += 1
fld_list = self.common_fields
fq_list = []
val_list = []
for f in fld_list:
fq_list.append(skytools.quote_ident(f))
v = unescape(src_row[f])
val_list.append(skytools.quote_literal(v))
q = "insert into %s (%s) values (%s);" % (
tbl, ", ".join(fq_list), ", ".join(val_list))
self.show_fix(tbl, q, 'insert')
def _new_obj_sql(self, queue, objname, objkind):
args = {'queue': skytools.quote_literal(queue),
'obj': objname,
'ifield': objname + '_name',
'itable': 'londiste.' + objname + '_info',
'kind': skytools.quote_literal(objkind),
}
q = """select quote_ident(n.nspname) || '.' || quote_ident(r.relname)
from pg_catalog.pg_class r
join pg_catalog.pg_namespace n on (n.oid = r.relnamespace)
left join %(itable)s i
on (i.queue_name = %(queue)s and
i.%(ifield)s = (n.nspname || '.' || r.relname))
where r.relkind = %(kind)s
and n.nspname not in ('pg_catalog', 'information_schema', 'pgq', 'londiste', 'pgq_node', 'pgq_ext')
and n.nspname !~ 'pg_.*'
and i.%(ifield)s is null
union all
select londiste.quote_fqname(%(ifield)s) from %(itable)s
where queue_name = %(queue)s and not local
order by 1 """ % args
return q
def got_missed_insert(self, src_row, fn):
"""Create sql for missed insert."""
self.cnt_insert += 1
fld_list = self.common_fields
fq_list = []
val_list = []
for f in fld_list:
fq_list.append(skytools.quote_ident(f))
v = skytools.unescape_copy(src_row[f])
val_list.append(skytools.quote_literal(v))
q = "insert into %s (%s) values (%s);" % (
self.fq_table_name, ", ".join(fq_list), ", ".join(val_list))
self.show_fix(q, 'insert', fn)
def mk_delete_sql(row, tbl, pkey_list, field_map = None):
"""Generate DELETE statement from dict data."""
if len(pkey_list) < 1:
raise Exception("delete needs pkeys")
whe_list = []
for k in pkey_list:
new_k = field_map and field_map[k] or k
col = skytools.quote_ident(new_k)
val = skytools.quote_literal(row[k])
whe_list.append("%s = %s" % (col, val))
whe_str = " and ".join(whe_list)
return "delete from only %s where %s;" % (skytools.quote_fqident(tbl), whe_str)
def mk_delete_sql(row, tbl, pkey_list, field_map=None):
"""Generate DELETE statement from dict data."""
if len(pkey_list) < 1:
raise Exception("delete needs pkeys")
whe_list = []
for k in pkey_list:
new_k = field_map and field_map[k] or k
col = skytools.quote_ident(new_k)
val = skytools.quote_literal(row[k])
whe_list.append("%s = %s" % (col, val))
whe_str = " and ".join(whe_list)
return "delete from only %s where %s;" % (skytools.quote_fqident(tbl),
whe_str)
def __init__(self, curs, seq_name):
"""Initializes class by loading info about table_name from database."""
super(SeqStruct, self).__init__(curs, seq_name)
# fill args
args = {
'fqname': self.fqname,
'fq2name': skytools.quote_literal(self.fqname),
'owner': 'null',
}
# load table struct
self.object_list = self._load_elem(curs, seq_name, args, TSeq)
def process_event(self, ev, sql_queue_func, qfunc_arg):
"""Ignore events for this table"""
fn = self.args.get('func_name')
fnconf = self.args.get('func_conf', '')
args = [
fnconf, self.cur_tick, ev.ev_id, ev.ev_time, ev.ev_txid,
ev.ev_retry, ev.ev_type, ev.ev_data, ev.ev_extra1, ev.ev_extra2,
ev.ev_extra3, ev.ev_extra4
]
qfn = skytools.quote_fqident(fn)
qargs = [skytools.quote_literal(a) for a in args]
sql = "select %s(%s);" % (qfn, ', '.join(qargs))
self.log.debug('applyfn.sql: %s' % sql)
sql_queue_func(sql, qfunc_arg)
def process_event(self, ev, sql_queue_func, qfunc_arg):
"""Ignore events for this table"""
fn = self.args.get('func_name')
fnconf = self.args.get('func_conf', '')
args = [fnconf, self.cur_tick,
ev.ev_id, ev.ev_time,
ev.ev_txid, ev.ev_retry,
ev.ev_type, ev.ev_data,
ev.ev_extra1, ev.ev_extra2,
ev.ev_extra3, ev.ev_extra4]
qfn = skytools.quote_fqident(fn)
qargs = [skytools.quote_literal(a) for a in args]
sql = "select %s(%s);" % (qfn, ', '.join(qargs))
self.log.debug('applyfn.sql: %s', sql)
sql_queue_func(sql, qfunc_arg)
def reload(self):
skytools.DBScript.reload(self)
self.pgq_lazy_fetch = self.cf.getint("pgq_lazy_fetch", self.default_lazy_fetch)
# set following ones to None if not set
self.pgq_min_count = self.cf.getint("pgq_batch_collect_events", 0) or None
self.pgq_min_interval = self.cf.get("pgq_batch_collect_interval", '') or None
self.pgq_min_lag = self.cf.get("pgq_keep_lag", '') or None
# filter out specific tables only
tfilt = []
for t in self.cf.getlist('table_filter', ''):
tfilt.append(skytools.quote_literal(skytools.fq_name(t)))
if len(tfilt) > 0:
expr = "ev_extra1 in (%s)" % ','.join(tfilt)
self.consumer_filter = expr
self.keepalive_stats = self.cf.getint("keepalive_stats", 300)
def mk_delete_sql(row, tbl, pkey_list, field_map=None):
"""Generate DELETE statement from dict data.
>>> mk_delete_sql({'a': 1, 'b':2, 'c':3}, 'tablename', ['a','b'])
"delete from only public.tablename where a = '1' and b = '2';"
>>> mk_delete_sql({'a': 1, 'b':2, 'c':3}, 'tablename', ['a','b'], {'a': 'aa', 'b':'bb'})
"delete from only public.tablename where aa = '1' and bb = '2';"
>>> mk_delete_sql({'a': 1, 'b':2, 'c':3}, 'tablename', [])
Traceback (most recent call last):
...
Exception: delete needs pkeys
"""
if len(pkey_list) < 1:
raise Exception("delete needs pkeys")
whe_list = []
for k in pkey_list:
new_k = field_map and field_map[k] or k
col = skytools.quote_ident(new_k)
val = skytools.quote_literal(row[k])
whe_list.append("%s = %s" % (col, val))
whe_str = " and ".join(whe_list)
return "delete from only %s where %s;" % (skytools.quote_fqident(tbl), whe_str)
def addeq(self, list, f, v):
vq = skytools.quote_literal(v)
s = "%s = %s" % (f, vq)
list.append(s)
def addeq(self, dst_list, f, v):
"""Add quoted SET."""
vq = skytools.quote_literal(v)
s = "%s = %s" % (f, vq)
dst_list.append(s)
def addeq(self, list, f, v):
"""Add quoted SET."""
vq = skytools.quote_literal(v)
s = "%s = %s" % (f, vq)
list.append(s)
