def get_upr_conn(self):
"""Return previously connected upr conn."""
if not self.upr_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
version = self.source_node['version']
logging.debug(" UPRStreamSource connecting mc: " + host + ":" +
str(port))
self.upr_conn = cb_bin_client.MemcachedClient(host, port)
if not self.upr_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
self.mem_conn = cb_bin_client.MemcachedClient(host, port)
if not self.upr_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
sasl_user = str(
self.source_bucket.get("name", self.opts.username) or "")
sasl_pswd = str(
self.source_bucket.get("saslPassword", self.opts.password)
or "")
if sasl_user:
try:
#self.upr_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
#self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
self.upr_conn.sasl_auth_plain(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
extra = struct.pack(couchbaseConstants.UPR_CONNECT_PKT_FMT, 0,
couchbaseConstants.FLAG_UPR_PRODUCER)
try:
opaque = self.r.randint(0, 2**32)
self.upr_conn._sendCmd(couchbaseConstants.CMD_UPR_CONNECT,
self.upr_name, '', opaque, extra)
self.upr_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up UPR connection", None
except cb_bin_client.MemcachedError:
return "error: UPR connect memcached error", None
except socket.error:
return "error: UPR connection error", None
self.running = True
self.start()
self.setup_upr_streams()
return 0, self.upr_conn
def get_tap_conn(self):
"""Return previously connected tap conn."""
if not self.tap_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
version = self.source_node['version'] # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'.
logging.debug(" TAPDumpSource connecting mc: " +
host + ":" + str(port))
self.tap_conn = cb_bin_client.MemcachedClient(host, port)
if not self.tap_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
if sasl_user:
try:
self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
try:
self.tap_conn.hello()
except Exception, e:
logging.warn("fail to call hello command, maybe it is not supported.")
pass
self.add_start_event(self.tap_conn)
# We explicitly do not use TAP_FLAG_REGISTERED_CLIENT,
# as that is for checkpoint/incremental backup only.
#
tap_opts = {couchbaseConstants.TAP_FLAG_DUMP: '',
couchbaseConstants.TAP_FLAG_SUPPORT_ACK: ''}
self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
if self.tap_conn.tap_fix_flag_byteorder:
tap_opts[couchbaseConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ''
if self.vbucket_list:
tap_opts[couchbaseConstants.TAP_FLAG_LIST_VBUCKETS] = ''
ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts, vblist=self.vbucket_list)
self.tap_conn._sendCmd(couchbaseConstants.CMD_TAP_CONNECT,
self.tap_name, val, 0, ext)
def get_mcd_conn(host, port, username, password, bucket):
conn = cb_bin_client.MemcachedClient(host, port)
if not conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
try:
conn.sasl_auth_plain(username, password)
except EOFError, e:
return "error: SASL auth error: %s:%s, %s" % (host, port, e), None
def get_mcd_conn(host,
port,
username,
password,
bucket,
use_ssl=False,
verify=True,
ca_cert=None,
collections=False):
conn = cb_bin_client.MemcachedClient(host,
port,
use_ssl=use_ssl,
verify=verify,
cacert=ca_cert)
if not conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
try:
conn.sasl_auth_plain(username, password)
except EOFError as e:
return "error: SASL auth error: %s:%s, %s" % (host, port, e), None
except cb_bin_client.MemcachedError as e:
return "error: SASL auth failed: %s:%s, %s" % (host, port, e), None
except socket.error as e:
return "error: SASL auth socket error: %s:%s, %s" % (host, port,
e), None
features = [couchbaseConstants.HELO_XATTR, couchbaseConstants.HELO_XERROR]
if collections:
features.append(couchbaseConstants.HELO_COLLECTIONS)
try:
conn.helo(features)
except EOFError as e:
return "error: HELO error: %s:%s, %s" % (host, port, e), None
except cb_bin_client.MemcachedError as e:
return "error: HELO failed: %s:%s, %s" % (host, port, e), None
except socket.error as e:
return "error: HELO socket error: %s:%s, %s" % (host, port, e), None
if bucket:
try:
conn.bucket_select(bucket)
except EOFError as e:
return "error: Bucket select error: %s:%s %s, %s" % (
host, port, bucket, e), None
except cb_bin_client.MemcachedError as e:
return "error: Bucket select failed: %s:%s %s, %s" % (
host, port, bucket, e), None
except socket.error as e:
return "error: Bucket select socket error: %s:%s %s, %s" % (
host, port, bucket, e), None
return 0, conn
def get_mcd_conn(host: str, port: int, username: str, password: str, bucket: Optional[str], use_ssl: bool = False,
verify: bool = True, ca_cert: Optional[str] = None, collections: bool = False) -> \
Tuple[couchbaseConstants.PUMP_ERROR, Optional[cb_bin_client.MemcachedClient]]:
conn = cb_bin_client.MemcachedClient(host,
port,
use_ssl=use_ssl,
verify=verify,
cacert=ca_cert)
if not conn:
return f'error: could not connect to memcached: {host}:{port!s}', None
try:
conn.sasl_auth_plain(username, password)
except EOFError as e:
return f'error: SASL auth error: {host}:{port}, {e}', None
except cb_bin_client.MemcachedError as e:
return f'error: SASL auth failed: {host}:{port}, {e}', None
except socket.error as e:
return f'error: SASL auth socket error: {host}:{port}, {e}', None
features = [
couchbaseConstants.HELO_XATTR, couchbaseConstants.HELO_XERROR,
couchbaseConstants.HELO_DATATYPE_COMPRESSED,
couchbaseConstants.HELO_DATATYPE_JSON
]
if collections:
features.append(couchbaseConstants.HELO_COLLECTIONS)
try:
_, _, enabled_features = conn.helo(features)
for feature in features:
if feature not in enabled_features:
return f'error: HELO denied feature:{feature} {host}:{port}', None
except EOFError as e:
return f'error: HELO error: {host}:{port}, {e}', None
except cb_bin_client.MemcachedError as e:
return f'error: HELO failed:{host}:{port}, {e}', None
except socket.error as e:
return f'error: HELO socket error: {host}:{port}, {e}', None
if bucket:
try:
conn.bucket_select(bucket.encode())
except EOFError as e:
return f'error: Bucket select error: {host}:{port} {bucket}, {e}', None
except cb_bin_client.MemcachedError as e:
return f'error: Bucket select failed: {host}:{port} {bucket}, {e}', None
except socket.error as e:
return f'error: Bucket select socket error: {host}:{port} {bucket}, {e}', None
return 0, conn
def get_mc_stats(self, server, bucketlist, nodes):
#print util.pretty_print(bucketlist)
for bucket in bucketlist:
bucket_name = bucket['name']
stats_buffer.node_stats[bucket_name] = {}
for node in nodes:
(node_server, node_port) = util.hostport(node['hostname'])
self.log.info(" node: %s %s" % (node_server, node['ports']['direct']))
stats = {}
mc = cb_bin_client.MemcachedClient(node_server, node['ports']['direct'])
if bucket["name"] != "Default":
mc.sasl_auth_cram_md5(bucket_name.encode("utf8"), bucket["saslPassword"].encode("utf8"))
self.get_mc_stats_per_node(mc, stats)
stats_buffer.node_stats[bucket_name][node['hostname']] = stats
def connect_mc(host, port, user, pswd):
mc = cb_bin_client.MemcachedClient(host, int(port))
if user:
try:
mc.sasl_auth_cram_md5(str(user), str(pswd))
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, user), None
except socket.error:
return "error: SASL auth exception: %s:%s, user: %s" % \
(host, port, user), None
return 0, mc
def connect_mc(self, node, sasl_user, sasl_pswd):
"""Return previously connected dcp conn."""
host = node.split(':')[0]
port = node.split(':')[1]
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
logging.debug(" DCPSink connecting mc: " + host + ":" + str(port))
dcp_conn = cb_bin_client.MemcachedClient(host, port)
if not dcp_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
if sasl_user:
try:
dcp_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
extra = struct.pack(couchbaseConstants.DCP_CONNECT_PKT_FMT, 0, \
couchbaseConstants.FLAG_DCP_CONSUMER)
try:
opaque = self.r.randint(0, 2**32)
dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONNECT,
self.dcp_name, '', opaque, extra)
dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection", None
except cb_bin_client.MemcachedError:
return "error: DCP connect memcached error", None
except socket.error:
return "error: DCP connection error", None
return 0, dcp_conn
def get_dcp_conn(self):
"""Return previously connected dcp conn."""
if not self.dcp_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
version = self.source_node['version']
logging.debug(" DCPStreamSource connecting mc: " + host + ":" +
str(port))
self.dcp_conn = cb_bin_client.MemcachedClient(host, port)
if not self.dcp_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port)
self.mem_conn = cb_bin_client.MemcachedClient(host, port)
if not self.mem_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port)
sasl_user = str(self.source_bucket.get("name"))
sasl_pswd = str(self.source_bucket.get("saslPassword"))
if sasl_user:
try:
self.dcp_conn.sasl_auth_plain(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user)
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user)
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user)
extra = struct.pack(couchbaseConstants.DCP_CONNECT_PKT_FMT, 0, \
couchbaseConstants.FLAG_DCP_PRODUCER)
try:
opaque = self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONNECT, self.dcp_name, \
'', opaque, extra)
self.dcp_conn._handleSingleResponse(opaque)
buff_size = 0
if self.flow_control:
# set connection buffer size. Considering header size, we roughly
# set the total received package size as 10 times as value size.
buff_size = self.batch_max_bytes * 10
opaque = self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(
couchbaseConstants.CMD_DCP_CONTROL,
couchbaseConstants.KEY_DCP_CONNECTION_BUFFER_SIZE,
str(self.batch_max_bytes * 10), opaque)
self.dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection"
except cb_bin_client.MemcachedError:
return "error: DCP connect memcached error"
except socket.error:
return "error: DCP connection error"
try:
opaque = self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
couchbaseConstants.KEY_DCP_EXT_METADATA,
bool_to_str(True), opaque)
self.dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection"
except cb_bin_client.MemcachedError:
pass
except socket.error:
return "error: DCP connection error"
self.running = True
self.start()
self.add_start_event(self.dcp_conn)
self.setup_dcp_streams()
return 0
