def _write_hook(_, data): if data[-1] == '\n': data = data[:-1] else: self.log.warning('Unexpected line from copy without end of line.') vals = data.split('\t') obf_vals = [] for field, value in zip(column_list, vals): obf_col = obf_col_map.get(field, {}) action = obf_col.get('action', actions.HASH) if action == actions.KEEP: obf_vals.append(value) continue str_val = skytools.unescape_copy(value) if str_val is None: obf_vals.append(value) continue if action == actions.HASH: obf_val = hash_function(str_val) obf_vals.append('%d' % obf_val) elif action == actions.JSON: obf_val = self.obf_json(str_val, obf_col) obf_vals.append(skytools.quote_copy(obf_val)) else: raise ValueError('Invalid value for action: %s' % action) obf_data = '\t'.join(obf_vals) + '\n' return obf_data
def got_missed_delete(self, dst_row, fn): """Create sql for missed delete.""" self.cnt_delete += 1 whe_list = [] for f in self.pkey_list: self.addcmp(whe_list, skytools.quote_ident(f), skytools.unescape_copy(dst_row[f])) q = "delete from only %s where %s;" % (self.fq_table_name, " and ".join(whe_list)) self.show_fix(q, 'delete', fn)
def data_to_dict(data, column_list): """Convert data received from copy to dict """ if data[-1] == '\n': data = data[:-1] vals = [skytools.unescape_copy(value) for value in data.split('\t')] row = dict(zip(column_list, vals)) return row
def got_missed_update(self, src_row, dst_row, fn): """Create sql for missed update.""" self.cnt_update += 1 fld_list = self.common_fields set_list = [] whe_list = [] for f in self.pkey_list: self.addcmp(whe_list, skytools.quote_ident(f), skytools.unescape_copy(src_row[f])) for f in fld_list: v1 = src_row[f] v2 = dst_row[f] if self.cmp_value(v1, v2) == 0: continue self.addeq(set_list, skytools.quote_ident(f), skytools.unescape_copy(v1)) self.addcmp(whe_list, skytools.quote_ident(f), skytools.unescape_copy(v2)) q = "update only %s set %s where %s;" % ( self.fq_table_name, ", ".join(set_list), " and ".join(whe_list)) self.show_fix(q, 'update', fn)
def got_missed_insert(self, src_row, fn): """Create sql for missed insert.""" self.cnt_insert += 1 fld_list = self.common_fields fq_list = [] val_list = [] for f in fld_list: fq_list.append(skytools.quote_ident(f)) v = skytools.unescape_copy(src_row[f]) val_list.append(skytools.quote_literal(v)) q = "insert into %s (%s) values (%s);" % ( self.fq_table_name, ", ".join(fq_list), ", ".join(val_list)) self.show_fix(q, 'insert', fn)
def unescape(s): """Remove copy escapes.""" return skytools.unescape_copy(s)
def unescape(s): return skytools.unescape_copy(s)