def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list):
"""do actual table copy and return tuple with number of bytes and rows
copyed
"""
condition = ' and '.join(cond_list)
return skytools.full_copy(tablename, src_curs, dst_curs, column_list,
condition)
def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list):
"""do actual table copy and return tuple with number of bytes and rows
copyed
"""
_src_cols = _dst_cols = column_list
condition = ' and '.join(cond_list)
if self.conf.skip_fields:
_src_cols = [col for col in column_list
if col not in self.conf.skip_fields]
_dst_cols = _src_cols
if self.conf.field_map:
_src_cols = [col for col in _src_cols if col in self.conf.field_map]
_dst_cols = [self.conf.field_map[col] for col in _src_cols]
if self.encoding_validator:
def _write_hook(obj, data):
return self.encoding_validator.validate_copy(data, _src_cols, tablename)
else:
_write_hook = None
return skytools.full_copy(tablename, src_curs, dst_curs, _src_cols,
condition, self.table_name, _dst_cols,
write_hook = _write_hook)
def real_copy(self, tablename, src_curs, dst_curs, column_list):
"""do actual table copy and return tuple with number of bytes and rows
copyed
"""
_src_cols = _dst_cols = column_list
condition = ''
if self.conf.skip_fields:
_src_cols = [
col for col in column_list if col not in self.conf.skip_fields
]
_dst_cols = _src_cols
if self.conf.field_map:
_src_cols = [
col for col in _src_cols if col in self.conf.field_map
]
_dst_cols = [self.conf.field_map[col] for col in _src_cols]
if self.encoding_validator:
def _write_hook(obj, data):
return self.encoding_validator.validate_copy(
data, _src_cols, tablename)
else:
_write_hook = None
return skytools.full_copy(tablename,
src_curs,
dst_curs,
_src_cols,
condition,
dst_tablename=self.dest_table,
dst_column_list=_dst_cols,
write_hook=_write_hook)
def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
"""do actual table copy and return tuple with number of bytes and rows
copied
"""
condition = self.get_copy_condition(src_curs, dst_curs)
return skytools.full_copy(src_tablename, src_curs, dst_curs,
column_list, condition,
dst_tablename=self.dest_table)
def real_copy(self, srccurs, dstcurs, tbl_stat, col_list):
"Actual copy."
tablename = tbl_stat.name
# do copy
self.log.info("%s: start copy" % tablename)
stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list)
if stats:
self.log.info("%s: copy finished: %d bytes, %d rows" % (
tablename, stats[0], stats[1]))
def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
"""do actual table copy and return tuple with number of bytes and rows
copyed
"""
condition = self.get_copy_condition(src_curs, dst_curs)
return skytools.full_copy(src_tablename,
src_curs,
dst_curs,
column_list,
condition,
dst_tablename=self.dest_table)
def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
"""do actual table copy and return tuple with number of bytes and rows
copied
"""
if self.encoding_validator:
def _write_hook(obj, data):
return self.encoding_validator.validate_copy(data, column_list, src_tablename)
else:
_write_hook = None
condition = self.get_copy_condition(src_curs, dst_curs)
return skytools.full_copy(src_tablename, src_curs, dst_curs,
column_list, condition,
dst_tablename=self.dest_table,
write_hook=_write_hook)
def real_copy(self, srccurs, dstcurs, tbl_stat, col_list):
"Actual copy."
tablename = tbl_stat.name
# do copy
self.log.info("%s: start copy" % tablename)
p = tbl_stat.get_plugin()
cond_list = []
cond = tbl_stat.table_attrs.get('copy_condition')
if cond:
cond_list.append(cond)
p.prepare_copy(cond_list, dstcurs)
w_cond = ' and '.join(cond_list)
stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list, w_cond)
if stats:
self.log.info("%s: copy finished: %d bytes, %d rows" % (
tablename, stats[0], stats[1]))
def real_copy(self, srccurs, dstcurs, tbl_stat, col_list):
"Main copy logic."
tablename = tbl_stat.name
# drop data
if tbl_stat.skip_truncate:
self.log.info("%s: skipping truncate" % tablename)
else:
self.log.info("%s: truncating" % tablename)
dstcurs.execute("truncate " + tablename)
# do copy
self.log.info("%s: start copy" % tablename)
stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list)
if stats:
self.log.info("%s: copy finished: %d bytes, %d rows" % (
tablename, stats[0], stats[1]))
def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
"""do actual table copy and return tuple with number of bytes and rows
copyed
"""
if self.enc:
def _write_hook(obj, data):
return self.enc.validate_copy(data, column_list, src_tablename)
else:
_write_hook = None
condition = self.get_copy_condition(src_curs, dst_curs)
return skytools.full_copy(src_tablename,
src_curs,
dst_curs,
column_list,
condition,
dst_tablename=self.dest_table,
write_hook=_write_hook)
def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
"""Initial copy
"""
self._validate(src_tablename, column_list)
obf_col_map = self.obf_map[src_tablename]
def _write_hook(_, data):
if data[-1] == '\n':
data = data[:-1]
else:
self.log.warning('Unexpected line from copy without end of line.')
vals = data.split('\t')
obf_vals = []
for field, value in zip(column_list, vals):
obf_col = obf_col_map.get(field, {})
action = obf_col.get('action', actions.HASH)
if action == actions.KEEP:
obf_vals.append(value)
continue
str_val = skytools.unescape_copy(value)
if str_val is None:
obf_vals.append(value)
continue
if action == actions.HASH:
obf_val = hash_function(str_val)
obf_vals.append('%d' % obf_val)
elif action == actions.JSON:
obf_val = self.obf_json(str_val, obf_col)
obf_vals.append(skytools.quote_copy(obf_val))
else:
raise ValueError('Invalid value for action: %s' % action)
obf_data = '\t'.join(obf_vals) + '\n'
return obf_data
condition = self.get_copy_condition(src_curs, dst_curs)
return skytools.full_copy(src_tablename, src_curs, dst_curs,
column_list, condition,
dst_tablename=self.dest_table,
write_hook=_write_hook)
def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
"""Initial copy
"""
obf_col_map = self._get_map(src_tablename)
new_list = []
for col in column_list:
action = obf_col_map.get(col, SKIP)
if action != SKIP:
new_list.append(col)
column_list = new_list
def _write_hook(_, data):
return self.obf_copy_row(data, column_list, src_tablename)
condition = self.get_copy_condition(src_curs, dst_curs)
return skytools.full_copy(src_tablename,
src_curs,
dst_curs,
column_list,
condition,
dst_tablename=self.dest_table,
write_hook=_write_hook)