예제 #1
0
 def add_stop_event(
     self, conn: Optional[cb_bin_client.MemcachedClient]
 ) -> couchbaseConstants.PUMP_ERROR:
     sasl_user = str(
         self.source_bucket.get("name",
                                pump.get_username(self.opts.username)))
     event = {
         "timestamp":
         self.get_timestamp(),
         "real_userid": {
             "source": "internal",
             "user": pump.return_string(sasl_user)
         },
         "source_bucket":
         pump.return_string(self.source_bucket['name']),
         "source_node":
         pump.return_string(self.source_node['hostname']),
         "target_bucket":
         pump.return_string(self.sink_map['buckets'][0]['name'])
     }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_STOP,
                        json.dumps(event))
         except Exception as e:
             logging.warning(f'auditing error: {e}')
     return 0
예제 #2
0
    def get_tap_conn(self):
        """Return previously connected tap conn."""

        if not self.tap_conn:
            host = self.source_node['hostname'].split(':')[0]
            port = self.source_node['ports']['direct']
            if self.opts.ssl:
                port = couchbaseConstants.SSL_PORT
            version = self.source_node['version'] # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'.

            logging.debug("  TAPDumpSource connecting mc: " +
                          host + ":" + str(port))

            self.tap_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.tap_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port), None

            sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
            sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
            if sasl_user:
                try:
                    self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                except EOFError:
                    return "error: SASL auth error: %s:%s, user: %s" % \
                        (host, port, sasl_user), None
                except cb_bin_client.MemcachedError:
                    return "error: SASL auth failed: %s:%s, user: %s" % \
                        (host, port, sasl_user), None
                except socket.error:
                    return "error: SASL auth socket error: %s:%s, user: %s" % \
                        (host, port, sasl_user), None

            try:
                self.tap_conn.hello()
            except Exception, e:
                logging.warn("fail to call hello command, maybe it is not supported.")
                pass

            self.add_start_event(self.tap_conn)
            # We explicitly do not use TAP_FLAG_REGISTERED_CLIENT,
            # as that is for checkpoint/incremental backup only.
            #
            tap_opts = {couchbaseConstants.TAP_FLAG_DUMP: '',
                        couchbaseConstants.TAP_FLAG_SUPPORT_ACK: ''}

            self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
            if self.tap_conn.tap_fix_flag_byteorder:
                tap_opts[couchbaseConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ''

            if self.vbucket_list:
                tap_opts[couchbaseConstants.TAP_FLAG_LIST_VBUCKETS] = ''

            ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts, vblist=self.vbucket_list)
            self.tap_conn._sendCmd(couchbaseConstants.CMD_TAP_CONNECT,
                                   self.tap_name, val, 0, ext)
예제 #3
0
 def add_stop_event(self, conn):
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {"timestamp": self.get_timestamp(),
              "real_userid": {"source": "internal",
                              "user": sasl_user
                             },
              "source_bucket": self.source_bucket['name'],
              "source_node": self.source_node['hostname']
             }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_BACKUP_STOP, json.dumps(event))
         except Exception, e:
             logging.warn("auditing error: %s" % e)
예제 #4
0
 def add_stop_event(self, conn):
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {
         "timestamp": self.get_timestamp(),
         "real_userid": {"source": "internal", "user": sasl_user},
         "source_bucket": self.source_bucket["name"],
         "source_node": self.source_node["hostname"],
         "target_bucket": self.sink_map["buckets"][0]["name"],
     }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SINK_STOP, json.dumps(event))
         except Exception, e:
             logging.warn("auditing error: %s" % e)
예제 #5
0
    def get_tap_conn(self):
        """Return previously connected tap conn."""

        if not self.tap_conn:
            host = self.source_node["hostname"].split(":")[0]
            port = self.source_node["ports"]["direct"]
            if self.opts.ssl:
                port = couchbaseConstants.SSL_PORT
            version = self.source_node["version"]  # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'.

            logging.debug("  TAPDumpSource connecting mc: " + host + ":" + str(port))

            self.tap_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.tap_conn:
                return "error: could not connect to memcached: " + host + ":" + str(port), None

            sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
            sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
            if sasl_user:
                try:
                    self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                except EOFError:
                    return "error: SASL auth error: %s:%s, user: %s" % (host, port, sasl_user), None
                except cb_bin_client.MemcachedError:
                    return "error: SASL auth failed: %s:%s, user: %s" % (host, port, sasl_user), None
                except socket.error:
                    return "error: SASL auth socket error: %s:%s, user: %s" % (host, port, sasl_user), None

            try:
                self.tap_conn.hello()
            except Exception, e:
                logging.warn("fail to call hello command, maybe it is not supported.")
                pass

            self.add_start_event(self.tap_conn)
            # We explicitly do not use TAP_FLAG_REGISTERED_CLIENT,
            # as that is for checkpoint/incremental backup only.
            #
            tap_opts = {couchbaseConstants.TAP_FLAG_DUMP: "", couchbaseConstants.TAP_FLAG_SUPPORT_ACK: ""}

            self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
            if self.tap_conn.tap_fix_flag_byteorder:
                tap_opts[couchbaseConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ""

            if self.vbucket_list:
                tap_opts[couchbaseConstants.TAP_FLAG_LIST_VBUCKETS] = ""

            ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts, vblist=self.vbucket_list)
            self.tap_conn._sendCmd(couchbaseConstants.CMD_TAP_CONNECT, self.tap_name, val, 0, ext)
예제 #6
0
 def add_start_event(self, conn):
     return 0
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {
         "timestamp": self.get_timestamp(),
         "real_userid": {"source": "internal", "user": sasl_user},
         "mode": getattr(self.opts, "mode", "diff"),
         "source_bucket": self.source_bucket["name"],
         "source_node": self.source_node["hostname"],
     }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SOURCE_START, json.dumps(event))
         except Exception, e:
             logging.warn("auditing error: %s" % e)
예제 #7
0
 def add_stop_event(self, conn):
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {"timestamp": self.get_timestamp(),
              "real_userid": {"source": "internal",
                              "user": sasl_user
                             },
              "source_bucket": self.source_bucket['name'],
              "source_node": self.source_node['hostname'],
              "target_bucket": self.sink_map['buckets'][0]['name']
             }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SINK_STOP, json.dumps(event))
         except Exception, e:
             logging.warn("auditing error: %s" % e)
예제 #8
0
 def add_start_event(self, conn):
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {"timestamp": self.get_timestamp(),
              "real_userid": {"source": "internal",
                              "user": sasl_user,
                             },
              "mode": getattr(self.opts, "mode", "diff"),
              "source_bucket": self.source_bucket['name'],
              "source_node": self.source_node['hostname'],
              "target_bucket": self.sink_map['buckets'][0]['name']
             }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_START, json.dumps(event))
         except Exception, e:
             logging.warn("auditing error: %s" % e)
예제 #9
0
 def add_start_event(self, conn):
     return 0
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {"timestamp": self.get_timestamp(),
              "real_userid": {"source": "internal",
                              "user": sasl_user,
                             },
              "mode": getattr(self.opts, "mode", "diff"),
              "source_bucket": self.source_bucket['name'],
              "source_node": self.source_node['hostname']
             }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SOURCE_START, json.dumps(event))
         except Exception, e:
             logging.warn("auditing error: %s" % e)
예제 #10
0
 def add_stop_event(self, conn: Optional[cb_bin_client.MemcachedClient]) -> couchbaseConstants.PUMP_ERROR:
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {"timestamp": self.get_timestamp(),
              "real_userid": {"source": "internal",
                              "user": pump.returnString(sasl_user)
                             },
              "source_bucket": pump.returnString(self.source_bucket['name']),
              "source_node": pump.returnString(self.source_node['hostname']),
              "target_bucket": pump.returnString(self.sink_map['buckets'][0]['name'])
             }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_STOP, json.dumps(event))
         except Exception as e:
             logging.warning(f'auditing error: {e}')
     return 0
예제 #11
0
 def add_start_event(self, conn):
     sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
     event = {"timestamp": self.get_timestamp(),
              "real_userid": {"source": "internal",
                              "user": pump.returnString(sasl_user),
                             },
              "mode": getattr(self.opts, "mode", "diff"),
              "source_bucket": pump.returnString(self.source_bucket['name']),
              "source_node": pump.returnString(self.source_node['hostname']),
              "target_bucket": pump.returnString(self.sink_map['buckets'][0]['name'])
             }
     if conn:
         try:
             conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_START, json.dumps(event))
         except Exception as e:
             logging.warn("auditing error: %s" % e)
     return 0
예제 #12
0
    def get_upr_conn(self):
        """Return previously connected upr conn."""

        if not self.upr_conn:
            host = self.source_node['hostname'].split(':')[0]
            port = self.source_node['ports']['direct']
            version = self.source_node['version']

            logging.debug("  UPRStreamSource connecting mc: " +
                          host + ":" + str(port))

            self.upr_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.upr_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port), None
            self.mem_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.upr_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port), None
            sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
            sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
            if sasl_user:
                try:
                    self.upr_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
                    self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
                except cb_bin_client.MemcachedError:
                    try:
                        self.upr_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                        self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                    except EOFError:
                        return "error: SASL auth error: %s:%s, user: %s" % \
                            (host, port, sasl_user), None
                    except cb_bin_client.MemcachedError:
                        return "error: SASL auth failed: %s:%s, user: %s" % \
                            (host, port, sasl_user), None
                    except socket.error:
                        return "error: SASL auth socket error: %s:%s, user: %s" % \
                            (host, port, sasl_user), None
                except EOFError:
                    return "error: SASL auth error: %s:%s, user: %s" % \
                        (host, port, sasl_user), None
                except socket.error:
                    return "error: SASL auth socket error: %s:%s, user: %s" % \
                        (host, port, sasl_user), None
            extra = struct.pack(couchbaseConstants.UPR_CONNECT_PKT_FMT, 0, \
                                couchbaseConstants.FLAG_UPR_PRODUCER)
            try:
                opaque=self.r.randint(0, 2**32)
                self.upr_conn._sendCmd(couchbaseConstants.CMD_UPR_CONNECT, self.upr_name, \
                                       '', opaque, extra)
                self.upr_conn._handleSingleResponse(opaque)
            except EOFError:
                return "error: Fail to set up UPR connection", None
            except cb_bin_client.MemcachedError:
                return "error: UPR connect memcached error", None
            except socket.error:
                return "error: UPR connection error", None
            self.running = True
            self.start()

            self.setup_upr_streams()
        return 0, self.upr_conn
예제 #13
0
    def get_dcp_conn(self):
        """Return previously connected dcp conn."""

        if not self.dcp_conn:
            host = self.source_node['hostname'].split(':')[0]
            port = self.source_node['ports']['direct']
            if self.opts.ssl:
                port = couchbaseConstants.SSL_PORT
            version = self.source_node['version']

            logging.debug("  DCPStreamSource connecting mc: " +
                          host + ":" + str(port))

            self.dcp_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.dcp_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port)
            self.mem_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.mem_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port)
            sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
            sasl_pswd = str(self.source_bucket.get("saslPassword", pump.get_password(self.opts.password)))
            if sasl_user:
                try:
                    self.dcp_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
                    self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
                except cb_bin_client.MemcachedError:
                    try:
                        self.dcp_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                        self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                    except EOFError:
                        return "error: SASL auth error: %s:%s, user: %s" % \
                            (host, port, sasl_user)
                    except cb_bin_client.MemcachedError:
                        return "error: SASL auth failed: %s:%s, user: %s" % \
                            (host, port, sasl_user)
                    except socket.error:
                        return "error: SASL auth socket error: %s:%s, user: %s" % \
                            (host, port, sasl_user)
                except EOFError:
                    return "error: SASL auth error: %s:%s, user: %s" % \
                        (host, port, sasl_user)
                except socket.error:
                    return "error: SASL auth socket error: %s:%s, user: %s" % \
                        (host, port, sasl_user)
            extra = struct.pack(couchbaseConstants.DCP_CONNECT_PKT_FMT, 0, \
                                couchbaseConstants.FLAG_DCP_PRODUCER)
            try:
                opaque=self.r.randint(0, 2**32)
                self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONNECT, self.dcp_name, \
                                       '', opaque, extra)
                self.dcp_conn._handleSingleResponse(opaque)

                buff_size = 0
                if self.flow_control:
                    # set connection buffer size. Considering header size, we roughly
                    # set the total received package size as 10 times as value size.
                    buff_size = self.batch_max_bytes * 10

                opaque=self.r.randint(0, 2**32)
                self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
                                       couchbaseConstants.KEY_DCP_CONNECTION_BUFFER_SIZE,
                                       str(self.batch_max_bytes * 10), opaque)
                self.dcp_conn._handleSingleResponse(opaque)
            except EOFError:
                return "error: Fail to set up DCP connection"
            except cb_bin_client.MemcachedError:
                return "error: DCP connect memcached error"
            except socket.error:
                return "error: DCP connection error"
            try:
                opaque=self.r.randint(0, 2**32)
                self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
                                       couchbaseConstants.KEY_DCP_EXT_METADATA,
                                       bool_to_str(True), opaque)
                self.dcp_conn._handleSingleResponse(opaque)
            except EOFError:
                return "error: Fail to set up DCP connection"
            except cb_bin_client.MemcachedError:
                pass
            except socket.error:
                return "error: DCP connection error"

            self.running = True
            self.start()

            self.add_start_event(self.dcp_conn)
            self.setup_dcp_streams()
        return 0
예제 #14
0
    def get_dcp_conn(self):
        """Return previously connected dcp conn."""

        if not self.dcp_conn:
            host = self.source_node['hostname'].split(':')[0]
            port = self.source_node['ports']['direct']
            if self.opts.ssl:
                port = couchbaseConstants.SSL_PORT
            version = self.source_node['version']

            logging.debug("  DCPStreamSource connecting mc: " + host + ":" +
                          str(port))

            self.dcp_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.dcp_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port)
            self.mem_conn = cb_bin_client.MemcachedClient(host, port)
            if not self.mem_conn:
                return "error: could not connect to memcached: " + \
                    host + ":" + str(port)
            sasl_user = str(
                self.source_bucket.get("name",
                                       pump.get_username(self.opts.username)))
            sasl_pswd = str(
                self.source_bucket.get("saslPassword",
                                       pump.get_password(self.opts.password)))
            if sasl_user:
                try:
                    self.dcp_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
                    self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
                except cb_bin_client.MemcachedError:
                    try:
                        self.dcp_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                        self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
                    except EOFError:
                        return "error: SASL auth error: %s:%s, user: %s" % \
                            (host, port, sasl_user)
                    except cb_bin_client.MemcachedError:
                        return "error: SASL auth failed: %s:%s, user: %s" % \
                            (host, port, sasl_user)
                    except socket.error:
                        return "error: SASL auth socket error: %s:%s, user: %s" % \
                            (host, port, sasl_user)
                except EOFError:
                    return "error: SASL auth error: %s:%s, user: %s" % \
                        (host, port, sasl_user)
                except socket.error:
                    return "error: SASL auth socket error: %s:%s, user: %s" % \
                        (host, port, sasl_user)
            extra = struct.pack(couchbaseConstants.DCP_CONNECT_PKT_FMT, 0, \
                                couchbaseConstants.FLAG_DCP_PRODUCER)
            try:
                opaque = self.r.randint(0, 2**32)
                self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONNECT, self.dcp_name, \
                                       '', opaque, extra)
                self.dcp_conn._handleSingleResponse(opaque)

                buff_size = 0
                if self.flow_control:
                    # set connection buffer size. Considering header size, we roughly
                    # set the total received package size as 10 times as value size.
                    buff_size = self.batch_max_bytes * 10

                opaque = self.r.randint(0, 2**32)
                self.dcp_conn._sendCmd(
                    couchbaseConstants.CMD_DCP_CONTROL,
                    couchbaseConstants.KEY_DCP_CONNECTION_BUFFER_SIZE,
                    str(self.batch_max_bytes * 10), opaque)
                self.dcp_conn._handleSingleResponse(opaque)
            except EOFError:
                return "error: Fail to set up DCP connection"
            except cb_bin_client.MemcachedError:
                return "error: DCP connect memcached error"
            except socket.error:
                return "error: DCP connection error"
            try:
                opaque = self.r.randint(0, 2**32)
                self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
                                       couchbaseConstants.KEY_DCP_EXT_METADATA,
                                       bool_to_str(True), opaque)
                self.dcp_conn._handleSingleResponse(opaque)
            except EOFError:
                return "error: Fail to set up DCP connection"
            except cb_bin_client.MemcachedError:
                pass
            except socket.error:
                return "error: DCP connection error"

            self.running = True
            self.start()

            self.add_start_event(self.dcp_conn)
            self.setup_dcp_streams()
        return 0