Пример #1
0
 def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list):
     """do actual table copy and return tuple with number of bytes and rows
     copyed
     """
     condition = ' and '.join(cond_list)
     return skytools.full_copy(tablename, src_curs, dst_curs, column_list,
                               condition)
Пример #2
0
    def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list):
        """do actual table copy and return tuple with number of bytes and rows
        copyed
        """
        _src_cols = _dst_cols = column_list
        condition = ' and '.join(cond_list)

        if self.conf.skip_fields:
            _src_cols = [col for col in column_list
                         if col not in self.conf.skip_fields]
            _dst_cols = _src_cols

        if self.conf.field_map:
            _src_cols = [col for col in _src_cols if col in self.conf.field_map]
            _dst_cols = [self.conf.field_map[col] for col in _src_cols]

        if self.encoding_validator:
            def _write_hook(obj, data):
                return self.encoding_validator.validate_copy(data, _src_cols, tablename)
        else:
            _write_hook = None

        return skytools.full_copy(tablename, src_curs, dst_curs, _src_cols,
                                  condition, self.table_name, _dst_cols,
                                  write_hook = _write_hook)
Пример #3
0
    def real_copy(self, tablename, src_curs, dst_curs, column_list):
        """do actual table copy and return tuple with number of bytes and rows
        copyed
        """
        _src_cols = _dst_cols = column_list
        condition = ''

        if self.conf.skip_fields:
            _src_cols = [
                col for col in column_list if col not in self.conf.skip_fields
            ]
            _dst_cols = _src_cols

        if self.conf.field_map:
            _src_cols = [
                col for col in _src_cols if col in self.conf.field_map
            ]
            _dst_cols = [self.conf.field_map[col] for col in _src_cols]

        if self.encoding_validator:

            def _write_hook(obj, data):
                return self.encoding_validator.validate_copy(
                    data, _src_cols, tablename)
        else:
            _write_hook = None

        return skytools.full_copy(tablename,
                                  src_curs,
                                  dst_curs,
                                  _src_cols,
                                  condition,
                                  dst_tablename=self.dest_table,
                                  dst_column_list=_dst_cols,
                                  write_hook=_write_hook)
Пример #4
0
 def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
     """do actual table copy and return tuple with number of bytes and rows
     copied
     """
     condition = self.get_copy_condition(src_curs, dst_curs)
     return skytools.full_copy(src_tablename, src_curs, dst_curs,
                               column_list, condition,
                               dst_tablename=self.dest_table)
Пример #5
0
    def real_copy(self, srccurs, dstcurs, tbl_stat, col_list):
        "Actual copy."

        tablename = tbl_stat.name
        # do copy
        self.log.info("%s: start copy" % tablename)
        stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list)
        if stats:
            self.log.info("%s: copy finished: %d bytes, %d rows" % (
                          tablename, stats[0], stats[1]))
Пример #6
0
 def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
     """do actual table copy and return tuple with number of bytes and rows
     copyed
     """
     condition = self.get_copy_condition(src_curs, dst_curs)
     return skytools.full_copy(src_tablename,
                               src_curs,
                               dst_curs,
                               column_list,
                               condition,
                               dst_tablename=self.dest_table)
Пример #7
0
    def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
        """do actual table copy and return tuple with number of bytes and rows
        copied
        """

        if self.encoding_validator:
            def _write_hook(obj, data):
                return self.encoding_validator.validate_copy(data, column_list, src_tablename)
        else:
            _write_hook = None
        condition = self.get_copy_condition(src_curs, dst_curs)
        return skytools.full_copy(src_tablename, src_curs, dst_curs,
                                  column_list, condition,
                                  dst_tablename=self.dest_table,
                                  write_hook=_write_hook)
Пример #8
0
    def real_copy(self, srccurs, dstcurs, tbl_stat, col_list):
        "Actual copy."

        tablename = tbl_stat.name
        # do copy
        self.log.info("%s: start copy" % tablename)
        p = tbl_stat.get_plugin()
        cond_list = []
        cond = tbl_stat.table_attrs.get('copy_condition')
        if cond:
            cond_list.append(cond)
        p.prepare_copy(cond_list, dstcurs)
        w_cond = ' and '.join(cond_list)
        stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list, w_cond)
        if stats:
            self.log.info("%s: copy finished: %d bytes, %d rows" % (
                          tablename, stats[0], stats[1]))
Пример #9
0
    def real_copy(self, srccurs, dstcurs, tbl_stat, col_list):
        "Main copy logic."

        tablename = tbl_stat.name
        # drop data
        if tbl_stat.skip_truncate:
            self.log.info("%s: skipping truncate" % tablename)
        else:
            self.log.info("%s: truncating" % tablename)
            dstcurs.execute("truncate " + tablename)

        # do copy
        self.log.info("%s: start copy" % tablename)
        stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list)
        if stats:
            self.log.info("%s: copy finished: %d bytes, %d rows" % (
                          tablename, stats[0], stats[1]))
Пример #10
0
    def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
        """do actual table copy and return tuple with number of bytes and rows
        copyed
        """

        if self.enc:

            def _write_hook(obj, data):
                return self.enc.validate_copy(data, column_list, src_tablename)
        else:
            _write_hook = None
        condition = self.get_copy_condition(src_curs, dst_curs)
        return skytools.full_copy(src_tablename,
                                  src_curs,
                                  dst_curs,
                                  column_list,
                                  condition,
                                  dst_tablename=self.dest_table,
                                  write_hook=_write_hook)
Пример #11
0
    def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
        """Initial copy
        """
        self._validate(src_tablename, column_list)
        obf_col_map = self.obf_map[src_tablename]
        def _write_hook(_, data):
            if data[-1] == '\n':
                data = data[:-1]
            else:
                self.log.warning('Unexpected line from copy without end of line.')

            vals = data.split('\t')
            obf_vals = []
            for field, value in zip(column_list, vals):
                obf_col = obf_col_map.get(field, {})
                action = obf_col.get('action', actions.HASH)

                if action == actions.KEEP:
                    obf_vals.append(value)
                    continue
                str_val = skytools.unescape_copy(value)
                if str_val is None:
                    obf_vals.append(value)
                    continue
                if action == actions.HASH:
                    obf_val = hash_function(str_val)
                    obf_vals.append('%d' % obf_val)
                elif action == actions.JSON:
                    obf_val = self.obf_json(str_val, obf_col)
                    obf_vals.append(skytools.quote_copy(obf_val))
                else:
                    raise ValueError('Invalid value for action: %s' % action)
            obf_data = '\t'.join(obf_vals) + '\n'
            return obf_data

        condition = self.get_copy_condition(src_curs, dst_curs)
        return skytools.full_copy(src_tablename, src_curs, dst_curs,
                                  column_list, condition,
                                  dst_tablename=self.dest_table,
                                  write_hook=_write_hook)
Пример #12
0
    def real_copy(self, src_tablename, src_curs, dst_curs, column_list):
        """Initial copy
        """
        obf_col_map = self._get_map(src_tablename)

        new_list = []
        for col in column_list:
            action = obf_col_map.get(col, SKIP)
            if action != SKIP:
                new_list.append(col)
        column_list = new_list

        def _write_hook(_, data):
            return self.obf_copy_row(data, column_list, src_tablename)

        condition = self.get_copy_condition(src_curs, dst_curs)
        return skytools.full_copy(src_tablename,
                                  src_curs,
                                  dst_curs,
                                  column_list,
                                  condition,
                                  dst_tablename=self.dest_table,
                                  write_hook=_write_hook)