def _write_hook(_, data):
if data[-1] == '\n':
data = data[:-1]
else:
self.log.warning('Unexpected line from copy without end of line.')
vals = data.split('\t')
obf_vals = []
for field, value in zip(column_list, vals):
obf_col = obf_col_map.get(field, {})
action = obf_col.get('action', actions.HASH)
if action == actions.KEEP:
obf_vals.append(value)
continue
str_val = skytools.unescape_copy(value)
if str_val is None:
obf_vals.append(value)
continue
if action == actions.HASH:
obf_val = hash_function(str_val)
obf_vals.append('%d' % obf_val)
elif action == actions.JSON:
obf_val = self.obf_json(str_val, obf_col)
obf_vals.append(skytools.quote_copy(obf_val))
else:
raise ValueError('Invalid value for action: %s' % action)
obf_data = '\t'.join(obf_vals) + '\n'
return obf_data
def got_missed_delete(self, dst_row, fn):
"""Create sql for missed delete."""
self.cnt_delete += 1
whe_list = []
for f in self.pkey_list:
self.addcmp(whe_list, skytools.quote_ident(f), skytools.unescape_copy(dst_row[f]))
q = "delete from only %s where %s;" % (self.fq_table_name, " and ".join(whe_list))
self.show_fix(q, 'delete', fn)
def data_to_dict(data, column_list):
"""Convert data received from copy to dict
"""
if data[-1] == '\n':
data = data[:-1]
vals = [skytools.unescape_copy(value) for value in data.split('\t')]
row = dict(zip(column_list, vals))
return row
def got_missed_update(self, src_row, dst_row, fn):
"""Create sql for missed update."""
self.cnt_update += 1
fld_list = self.common_fields
set_list = []
whe_list = []
for f in self.pkey_list:
self.addcmp(whe_list, skytools.quote_ident(f), skytools.unescape_copy(src_row[f]))
for f in fld_list:
v1 = src_row[f]
v2 = dst_row[f]
if self.cmp_value(v1, v2) == 0:
continue
self.addeq(set_list, skytools.quote_ident(f), skytools.unescape_copy(v1))
self.addcmp(whe_list, skytools.quote_ident(f), skytools.unescape_copy(v2))
q = "update only %s set %s where %s;" % (
self.fq_table_name, ", ".join(set_list), " and ".join(whe_list))
self.show_fix(q, 'update', fn)
def got_missed_insert(self, src_row, fn):
"""Create sql for missed insert."""
self.cnt_insert += 1
fld_list = self.common_fields
fq_list = []
val_list = []
for f in fld_list:
fq_list.append(skytools.quote_ident(f))
v = skytools.unescape_copy(src_row[f])
val_list.append(skytools.quote_literal(v))
q = "insert into %s (%s) values (%s);" % (
self.fq_table_name, ", ".join(fq_list), ", ".join(val_list))
self.show_fix(q, 'insert', fn)
def got_missed_insert(self, src_row, fn):
"""Create sql for missed insert."""
self.cnt_insert += 1
fld_list = self.common_fields
fq_list = []
val_list = []
for f in fld_list:
fq_list.append(skytools.quote_ident(f))
v = skytools.unescape_copy(src_row[f])
val_list.append(skytools.quote_literal(v))
q = "insert into %s (%s) values (%s);" % (
self.fq_table_name, ", ".join(fq_list), ", ".join(val_list))
self.show_fix(q, 'insert', fn)
def unescape(s):
"""Remove copy escapes."""
return skytools.unescape_copy(s)
def unescape(s):
"""Remove copy escapes."""
return skytools.unescape_copy(s)
def unescape(s):
return skytools.unescape_copy(s)
