def process_event(self, ev, sql_queue_func, arg):
row = self.parse_row_data(ev)
if len(ev.type) == 1:
# sql event
fqname = self.fq_dest_table
fmt = self.sql_command[ev.type]
sql = fmt % (fqname, row)
else:
if ev.type[0] == '{':
jtype = json.loads(ev.type)
pklist = jtype['pkey']
op = jtype['op'][0]
else:
# urlenc event
pklist = ev.type[2:].split(',')
op = ev.type[0]
tbl = self.dest_table
if op == 'I':
sql = skytools.mk_insert_sql(row, tbl, pklist)
elif op == 'U':
sql = skytools.mk_update_sql(row, tbl, pklist)
elif op == 'D':
sql = skytools.mk_delete_sql(row, tbl, pklist)
sql_queue_func(sql, arg)
def process_event(self, ev, sql_queue_func, arg):
row = self.parse_row_data(ev)
if len(ev.type) == 1:
# sql event
fqname = self.fq_dest_table
fmt = self.sql_command[ev.type]
sql = fmt % (fqname, row)
else:
if ev.type[0] == '{':
jtype = json.loads(ev.type)
pklist = jtype['pkey']
op = jtype['op'][0]
else:
# urlenc event
pklist = ev.type[2:].split(',')
op = ev.type[0]
tbl = self.dest_table
if op == 'I':
sql = skytools.mk_insert_sql(row, tbl, pklist)
elif op == 'U':
sql = skytools.mk_update_sql(row, tbl, pklist)
elif op == 'D':
sql = skytools.mk_delete_sql(row, tbl, pklist)
sql_queue_func(sql, arg)
def add_row(self, op, data, pkey_list):
if op == 'I':
sql = skytools.mk_insert_sql(data, self.table_name, pkey_list)
elif op == 'U':
sql = skytools.mk_update_sql(data, self.table_name, pkey_list)
elif op == 'D':
sql = skytools.mk_delete_sql(data, self.table_name, pkey_list)
else:
raise Exception('bad operation: ' + op)
self.sql_list.append(sql)
def add_row(self, op, data, pkey_list):
if op == 'I':
sql = skytools.mk_insert_sql(data, self.table_name, pkey_list)
elif op == 'U':
sql = skytools.mk_update_sql(data, self.table_name, pkey_list)
elif op == 'D':
sql = skytools.mk_delete_sql(data, self.table_name, pkey_list)
else:
raise Exception('bad operation: '+op)
self.sql_list.append(sql)
def process_event(self, ev, sql_queue_func, arg):
if len(ev.type) == 1:
# sql event
fqname = skytools.quote_fqident(ev.extra1)
fmt = self.sql_command[ev.type]
sql = fmt % (fqname, ev.data)
else:
# urlenc event
pklist = ev.type[2:].split(',')
row = skytools.db_urldecode(ev.data)
op = ev.type[0]
tbl = ev.extra1
if op == 'I':
sql = skytools.mk_insert_sql(row, tbl, pklist)
elif op == 'U':
sql = skytools.mk_update_sql(row, tbl, pklist)
elif op == 'D':
sql = skytools.mk_delete_sql(row, tbl, pklist)
sql_queue_func(sql, arg)
def process_event(self, ev, sql_queue_func, arg):
row = self.parse_row_data(ev)
if len(ev.type) == 1:
# sql event
fqname = self.fq_dest_table
fmt = self.sql_command[ev.type]
sql = fmt % (fqname, row)
else:
# urlenc event
pklist = ev.type[2:].split(",")
op = ev.type[0]
tbl = self.dest_table
if op == "I":
sql = skytools.mk_insert_sql(row, tbl, pklist)
elif op == "U":
sql = skytools.mk_update_sql(row, tbl, pklist)
elif op == "D":
sql = skytools.mk_delete_sql(row, tbl, pklist)
sql_queue_func(sql, arg)
def handle_urlenc_event(self, ev, dst_curs):
"""handle one truncate event"""
t = self.get_table_by_name(ev.extra1)
if not t or not t.interesting(ev, self.cur_tick, self.copy_thread):
self.stat_increase('ignored_events')
return
# parse event
pklist = ev.type[2:].split(',')
row = skytools.db_urldecode(ev.data)
op = ev.type[0]
tbl = ev.extra1
# generate sql
if op == 'I':
sql = skytools.mk_insert_sql(row, tbl, pklist)
elif op == 'U':
sql = skytools.mk_update_sql(row, tbl, pklist)
elif op == 'D':
sql = skytools.mk_delete_sql(row, tbl, pklist)
else:
raise Exception('bug: bad op')
self.apply_sql(sql, dst_curs)
def applyrow(tblname, ev_type, new_row,
backup_row = None,
alt_pkey_cols = None,
fkey_cols = None,
fkey_ref_table = None,
fkey_ref_cols = None,
fn_canapply = canapply_dummy,
fn_colfilter = colfilter_full):
"""Core logic. Actual decisions will be done in callback functions.
- [IUD]: If row referenced by fkey does not exist, event is not applied
- If pkey does not exist but alt_pkey does, row is not applied.
@param tblname: table name, schema-qualified
@param ev_type: [IUD]:pkey1,pkey2
@param alt_pkey_cols: list of alternatice columns to consuder
@param fkey_cols: columns in this table that refer to other table
@param fkey_ref_table: other table referenced here
@param fkey_ref_cols: column in other table that must match
@param fn_canapply: callback function, gets new and old row, returns whether the row should be applied
@param fn_colfilter: callback function, gets new and old row, returns dict of final columns to be applied
"""
gd = None
# parse ev_type
tmp = ev_type.split(':', 1)
if len(tmp) != 2 or tmp[0] not in ('I', 'U', 'D'):
raise DataError('Unsupported ev_type: '+repr(ev_type))
if not tmp[1]:
raise DataError('No pkey in event')
cmd = tmp[0]
pkey_cols = tmp[1].split(',')
qtblname = skytools.quote_fqident(tblname)
# parse ev_data
fields = skytools.db_urldecode(new_row)
if ev_type.find('}') >= 0:
raise DataError('Really suspicious activity')
if ",".join(fields.keys()).find('}') >= 0:
raise DataError('Really suspicious activity 2')
# generate pkey expressions
tmp = ["%s = {%s}" % (skytools.quote_ident(k), k) for k in pkey_cols]
pkey_expr = " and ".join(tmp)
alt_pkey_expr = None
if alt_pkey_cols:
tmp = ["%s = {%s}" % (skytools.quote_ident(k), k) for k in alt_pkey_cols]
alt_pkey_expr = " and ".join(tmp)
log = "data ok"
#
# Row data seems fine, now apply it
#
if fkey_ref_table:
tmp = []
for k, rk in zip(fkey_cols, fkey_ref_cols):
tmp.append("%s = {%s}" % (skytools.quote_ident(rk), k))
fkey_expr = " and ".join(tmp)
q = "select 1 from only %s where %s" % (
skytools.quote_fqident(fkey_ref_table),
fkey_expr)
res = skytools.plpy_exec(gd, q, fields)
if not res:
return "IGN: parent row does not exist"
log += ", fkey ok"
# fetch old row
if alt_pkey_expr:
q = "select * from only %s where %s for update" % (qtblname, alt_pkey_expr)
res = skytools.plpy_exec(gd, q, fields)
if res:
oldrow = res[0]
# if altpk matches, but pk not, then delete
need_del = 0
for k in pkey_cols:
# fixme: proper type cmp?
if fields[k] != str(oldrow[k]):
need_del = 1
break
if need_del:
log += ", altpk del"
q = "delete from only %s where %s" % (qtblname, alt_pkey_expr)
skytools.plpy_exec(gd, q, fields)
res = None
else:
log += ", altpk ok"
else:
# no altpk
q = "select * from only %s where %s for update" % (qtblname, pkey_expr)
res = skytools.plpy_exec(None, q, fields)
# got old row, with same pk and altpk
if res:
oldrow = res[0]
log += ", old row"
ok = fn_canapply(fields, oldrow)
if ok:
log += ", new row better"
if not ok:
# ignore the update
return "IGN:" + log + ", current row more up-to-date"
else:
log += ", no old row"
oldrow = None
if res:
if cmd == 'I':
cmd = 'U'
else:
if cmd == 'U':
cmd = 'I'
# allow column changes
if oldrow:
fields2 = fn_colfilter(fields, oldrow)
for k in pkey_cols:
if k not in fields2:
fields2[k] = fields[k]
fields = fields2
# apply change
if cmd == 'I':
q = skytools.mk_insert_sql(fields, tblname, pkey_cols)
elif cmd == 'U':
q = skytools.mk_update_sql(fields, tblname, pkey_cols)
elif cmd == 'D':
q = skytools.mk_delete_sql(fields, tblname, pkey_cols)
else:
plpy.error('Huh')
plpy.execute(q)
return log
def applyrow(tblname,
ev_type,
new_row,
backup_row=None,
alt_pkey_cols=None,
fkey_cols=None,
fkey_ref_table=None,
fkey_ref_cols=None,
fn_canapply=canapply_dummy,
fn_colfilter=colfilter_full):
"""Core logic. Actual decisions will be done in callback functions.
- [IUD]: If row referenced by fkey does not exist, event is not applied
- If pkey does not exist but alt_pkey does, row is not applied.
@param tblname: table name, schema-qualified
@param ev_type: [IUD]:pkey1,pkey2
@param alt_pkey_cols: list of alternatice columns to consuder
@param fkey_cols: columns in this table that refer to other table
@param fkey_ref_table: other table referenced here
@param fkey_ref_cols: column in other table that must match
@param fn_canapply: callback function, gets new and old row, returns whether the row should be applied
@param fn_colfilter: callback function, gets new and old row, returns dict of final columns to be applied
"""
gd = None
# parse ev_type
tmp = ev_type.split(':', 1)
if len(tmp) != 2 or tmp[0] not in ('I', 'U', 'D'):
raise DataError('Unsupported ev_type: ' + repr(ev_type))
if not tmp[1]:
raise DataError('No pkey in event')
cmd = tmp[0]
pkey_cols = tmp[1].split(',')
qtblname = skytools.quote_fqident(tblname)
# parse ev_data
fields = skytools.db_urldecode(new_row)
if ev_type.find('}') >= 0:
raise DataError('Really suspicious activity')
if ",".join(fields.keys()).find('}') >= 0:
raise DataError('Really suspicious activity 2')
# generate pkey expressions
tmp = ["%s = {%s}" % (skytools.quote_ident(k), k) for k in pkey_cols]
pkey_expr = " and ".join(tmp)
alt_pkey_expr = None
if alt_pkey_cols:
tmp = [
"%s = {%s}" % (skytools.quote_ident(k), k) for k in alt_pkey_cols
]
alt_pkey_expr = " and ".join(tmp)
log = "data ok"
#
# Row data seems fine, now apply it
#
if fkey_ref_table:
tmp = []
for k, rk in zip(fkey_cols, fkey_ref_cols):
tmp.append("%s = {%s}" % (skytools.quote_ident(rk), k))
fkey_expr = " and ".join(tmp)
q = "select 1 from only %s where %s" % (
skytools.quote_fqident(fkey_ref_table), fkey_expr)
res = skytools.plpy_exec(gd, q, fields)
if not res:
return "IGN: parent row does not exist"
log += ", fkey ok"
# fetch old row
if alt_pkey_expr:
q = "select * from only %s where %s for update" % (qtblname,
alt_pkey_expr)
res = skytools.plpy_exec(gd, q, fields)
if res:
oldrow = res[0]
# if altpk matches, but pk not, then delete
need_del = 0
for k in pkey_cols:
# fixme: proper type cmp?
if fields[k] != str(oldrow[k]):
need_del = 1
break
if need_del:
log += ", altpk del"
q = "delete from only %s where %s" % (qtblname, alt_pkey_expr)
skytools.plpy_exec(gd, q, fields)
res = None
else:
log += ", altpk ok"
else:
# no altpk
q = "select * from only %s where %s for update" % (qtblname, pkey_expr)
res = skytools.plpy_exec(None, q, fields)
# got old row, with same pk and altpk
if res:
oldrow = res[0]
log += ", old row"
ok = fn_canapply(fields, oldrow)
if ok:
log += ", new row better"
if not ok:
# ignore the update
return "IGN:" + log + ", current row more up-to-date"
else:
log += ", no old row"
oldrow = None
if res:
if cmd == 'I':
cmd = 'U'
else:
if cmd == 'U':
cmd = 'I'
# allow column changes
if oldrow:
fields2 = fn_colfilter(fields, oldrow)
for k in pkey_cols:
if k not in fields2:
fields2[k] = fields[k]
fields = fields2
# apply change
if cmd == 'I':
q = skytools.mk_insert_sql(fields, tblname, pkey_cols)
elif cmd == 'U':
q = skytools.mk_update_sql(fields, tblname, pkey_cols)
elif cmd == 'D':
q = skytools.mk_delete_sql(fields, tblname, pkey_cols)
else:
plpy.error('Huh')
plpy.execute(q)
return log