def add_stop_event(
self, conn: Optional[cb_bin_client.MemcachedClient]
) -> couchbaseConstants.PUMP_ERROR:
sasl_user = str(
self.source_bucket.get("name",
pump.get_username(self.opts.username)))
event = {
"timestamp":
self.get_timestamp(),
"real_userid": {
"source": "internal",
"user": pump.return_string(sasl_user)
},
"source_bucket":
pump.return_string(self.source_bucket['name']),
"source_node":
pump.return_string(self.source_node['hostname']),
"target_bucket":
pump.return_string(self.sink_map['buckets'][0]['name'])
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_STOP,
json.dumps(event))
except Exception as e:
logging.warning(f'auditing error: {e}')
return 0
def get_tap_conn(self):
"""Return previously connected tap conn."""
if not self.tap_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
version = self.source_node['version'] # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'.
logging.debug(" TAPDumpSource connecting mc: " +
host + ":" + str(port))
self.tap_conn = cb_bin_client.MemcachedClient(host, port)
if not self.tap_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
if sasl_user:
try:
self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
try:
self.tap_conn.hello()
except Exception, e:
logging.warn("fail to call hello command, maybe it is not supported.")
pass
self.add_start_event(self.tap_conn)
# We explicitly do not use TAP_FLAG_REGISTERED_CLIENT,
# as that is for checkpoint/incremental backup only.
#
tap_opts = {couchbaseConstants.TAP_FLAG_DUMP: '',
couchbaseConstants.TAP_FLAG_SUPPORT_ACK: ''}
self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
if self.tap_conn.tap_fix_flag_byteorder:
tap_opts[couchbaseConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ''
if self.vbucket_list:
tap_opts[couchbaseConstants.TAP_FLAG_LIST_VBUCKETS] = ''
ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts, vblist=self.vbucket_list)
self.tap_conn._sendCmd(couchbaseConstants.CMD_TAP_CONNECT,
self.tap_name, val, 0, ext)
def add_stop_event(self, conn):
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal",
"user": sasl_user
},
"source_bucket": self.source_bucket['name'],
"source_node": self.source_node['hostname']
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_BACKUP_STOP, json.dumps(event))
except Exception, e:
logging.warn("auditing error: %s" % e)
def add_stop_event(self, conn):
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {
"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal", "user": sasl_user},
"source_bucket": self.source_bucket["name"],
"source_node": self.source_node["hostname"],
"target_bucket": self.sink_map["buckets"][0]["name"],
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SINK_STOP, json.dumps(event))
except Exception, e:
logging.warn("auditing error: %s" % e)
def get_tap_conn(self):
"""Return previously connected tap conn."""
if not self.tap_conn:
host = self.source_node["hostname"].split(":")[0]
port = self.source_node["ports"]["direct"]
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
version = self.source_node["version"] # Ex: '2.0.0-1944-rel-community' or '1.8.1-937-rel-community'.
logging.debug(" TAPDumpSource connecting mc: " + host + ":" + str(port))
self.tap_conn = cb_bin_client.MemcachedClient(host, port)
if not self.tap_conn:
return "error: could not connect to memcached: " + host + ":" + str(port), None
sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
if sasl_user:
try:
self.tap_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % (host, port, sasl_user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % (host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % (host, port, sasl_user), None
try:
self.tap_conn.hello()
except Exception, e:
logging.warn("fail to call hello command, maybe it is not supported.")
pass
self.add_start_event(self.tap_conn)
# We explicitly do not use TAP_FLAG_REGISTERED_CLIENT,
# as that is for checkpoint/incremental backup only.
#
tap_opts = {couchbaseConstants.TAP_FLAG_DUMP: "", couchbaseConstants.TAP_FLAG_SUPPORT_ACK: ""}
self.tap_conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
if self.tap_conn.tap_fix_flag_byteorder:
tap_opts[couchbaseConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ""
if self.vbucket_list:
tap_opts[couchbaseConstants.TAP_FLAG_LIST_VBUCKETS] = ""
ext, val = TAPDumpSource.encode_tap_connect_opts(tap_opts, vblist=self.vbucket_list)
self.tap_conn._sendCmd(couchbaseConstants.CMD_TAP_CONNECT, self.tap_name, val, 0, ext)
def add_start_event(self, conn):
return 0
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {
"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal", "user": sasl_user},
"mode": getattr(self.opts, "mode", "diff"),
"source_bucket": self.source_bucket["name"],
"source_node": self.source_node["hostname"],
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SOURCE_START, json.dumps(event))
except Exception, e:
logging.warn("auditing error: %s" % e)
def add_stop_event(self, conn):
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal",
"user": sasl_user
},
"source_bucket": self.source_bucket['name'],
"source_node": self.source_node['hostname'],
"target_bucket": self.sink_map['buckets'][0]['name']
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SINK_STOP, json.dumps(event))
except Exception, e:
logging.warn("auditing error: %s" % e)
def add_start_event(self, conn):
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal",
"user": sasl_user,
},
"mode": getattr(self.opts, "mode", "diff"),
"source_bucket": self.source_bucket['name'],
"source_node": self.source_node['hostname'],
"target_bucket": self.sink_map['buckets'][0]['name']
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_START, json.dumps(event))
except Exception, e:
logging.warn("auditing error: %s" % e)
def add_start_event(self, conn):
return 0
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal",
"user": sasl_user,
},
"mode": getattr(self.opts, "mode", "diff"),
"source_bucket": self.source_bucket['name'],
"source_node": self.source_node['hostname']
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RECOVERY_SOURCE_START, json.dumps(event))
except Exception, e:
logging.warn("auditing error: %s" % e)
def add_stop_event(self, conn: Optional[cb_bin_client.MemcachedClient]) -> couchbaseConstants.PUMP_ERROR:
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal",
"user": pump.returnString(sasl_user)
},
"source_bucket": pump.returnString(self.source_bucket['name']),
"source_node": pump.returnString(self.source_node['hostname']),
"target_bucket": pump.returnString(self.sink_map['buckets'][0]['name'])
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_STOP, json.dumps(event))
except Exception as e:
logging.warning(f'auditing error: {e}')
return 0
def add_start_event(self, conn):
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
event = {"timestamp": self.get_timestamp(),
"real_userid": {"source": "internal",
"user": pump.returnString(sasl_user),
},
"mode": getattr(self.opts, "mode", "diff"),
"source_bucket": pump.returnString(self.source_bucket['name']),
"source_node": pump.returnString(self.source_node['hostname']),
"target_bucket": pump.returnString(self.sink_map['buckets'][0]['name'])
}
if conn:
try:
conn.audit(couchbaseConstants.AUDIT_EVENT_RESTORE_SINK_START, json.dumps(event))
except Exception as e:
logging.warn("auditing error: %s" % e)
return 0
def get_upr_conn(self):
"""Return previously connected upr conn."""
if not self.upr_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
version = self.source_node['version']
logging.debug(" UPRStreamSource connecting mc: " +
host + ":" + str(port))
self.upr_conn = cb_bin_client.MemcachedClient(host, port)
if not self.upr_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
self.mem_conn = cb_bin_client.MemcachedClient(host, port)
if not self.upr_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port), None
sasl_user = str(pump.get_username(self.source_bucket.get("name", self.opts.username)))
sasl_pswd = str(pump.get_password(self.source_bucket.get("saslPassword", self.opts.password)))
if sasl_user:
try:
self.upr_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
except cb_bin_client.MemcachedError:
try:
self.upr_conn.sasl_auth_plain(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user), None
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user), None
extra = struct.pack(couchbaseConstants.UPR_CONNECT_PKT_FMT, 0, \
couchbaseConstants.FLAG_UPR_PRODUCER)
try:
opaque=self.r.randint(0, 2**32)
self.upr_conn._sendCmd(couchbaseConstants.CMD_UPR_CONNECT, self.upr_name, \
'', opaque, extra)
self.upr_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up UPR connection", None
except cb_bin_client.MemcachedError:
return "error: UPR connect memcached error", None
except socket.error:
return "error: UPR connection error", None
self.running = True
self.start()
self.setup_upr_streams()
return 0, self.upr_conn
def get_dcp_conn(self):
"""Return previously connected dcp conn."""
if not self.dcp_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
version = self.source_node['version']
logging.debug(" DCPStreamSource connecting mc: " +
host + ":" + str(port))
self.dcp_conn = cb_bin_client.MemcachedClient(host, port)
if not self.dcp_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port)
self.mem_conn = cb_bin_client.MemcachedClient(host, port)
if not self.mem_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port)
sasl_user = str(self.source_bucket.get("name", pump.get_username(self.opts.username)))
sasl_pswd = str(self.source_bucket.get("saslPassword", pump.get_password(self.opts.password)))
if sasl_user:
try:
self.dcp_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
except cb_bin_client.MemcachedError:
try:
self.dcp_conn.sasl_auth_plain(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user)
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user)
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user)
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user)
extra = struct.pack(couchbaseConstants.DCP_CONNECT_PKT_FMT, 0, \
couchbaseConstants.FLAG_DCP_PRODUCER)
try:
opaque=self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONNECT, self.dcp_name, \
'', opaque, extra)
self.dcp_conn._handleSingleResponse(opaque)
buff_size = 0
if self.flow_control:
# set connection buffer size. Considering header size, we roughly
# set the total received package size as 10 times as value size.
buff_size = self.batch_max_bytes * 10
opaque=self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
couchbaseConstants.KEY_DCP_CONNECTION_BUFFER_SIZE,
str(self.batch_max_bytes * 10), opaque)
self.dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection"
except cb_bin_client.MemcachedError:
return "error: DCP connect memcached error"
except socket.error:
return "error: DCP connection error"
try:
opaque=self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
couchbaseConstants.KEY_DCP_EXT_METADATA,
bool_to_str(True), opaque)
self.dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection"
except cb_bin_client.MemcachedError:
pass
except socket.error:
return "error: DCP connection error"
self.running = True
self.start()
self.add_start_event(self.dcp_conn)
self.setup_dcp_streams()
return 0
def get_dcp_conn(self):
"""Return previously connected dcp conn."""
if not self.dcp_conn:
host = self.source_node['hostname'].split(':')[0]
port = self.source_node['ports']['direct']
if self.opts.ssl:
port = couchbaseConstants.SSL_PORT
version = self.source_node['version']
logging.debug(" DCPStreamSource connecting mc: " + host + ":" +
str(port))
self.dcp_conn = cb_bin_client.MemcachedClient(host, port)
if not self.dcp_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port)
self.mem_conn = cb_bin_client.MemcachedClient(host, port)
if not self.mem_conn:
return "error: could not connect to memcached: " + \
host + ":" + str(port)
sasl_user = str(
self.source_bucket.get("name",
pump.get_username(self.opts.username)))
sasl_pswd = str(
self.source_bucket.get("saslPassword",
pump.get_password(self.opts.password)))
if sasl_user:
try:
self.dcp_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_cram_md5(sasl_user, sasl_pswd)
except cb_bin_client.MemcachedError:
try:
self.dcp_conn.sasl_auth_plain(sasl_user, sasl_pswd)
self.mem_conn.sasl_auth_plain(sasl_user, sasl_pswd)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user)
except cb_bin_client.MemcachedError:
return "error: SASL auth failed: %s:%s, user: %s" % \
(host, port, sasl_user)
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user)
except EOFError:
return "error: SASL auth error: %s:%s, user: %s" % \
(host, port, sasl_user)
except socket.error:
return "error: SASL auth socket error: %s:%s, user: %s" % \
(host, port, sasl_user)
extra = struct.pack(couchbaseConstants.DCP_CONNECT_PKT_FMT, 0, \
couchbaseConstants.FLAG_DCP_PRODUCER)
try:
opaque = self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONNECT, self.dcp_name, \
'', opaque, extra)
self.dcp_conn._handleSingleResponse(opaque)
buff_size = 0
if self.flow_control:
# set connection buffer size. Considering header size, we roughly
# set the total received package size as 10 times as value size.
buff_size = self.batch_max_bytes * 10
opaque = self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(
couchbaseConstants.CMD_DCP_CONTROL,
couchbaseConstants.KEY_DCP_CONNECTION_BUFFER_SIZE,
str(self.batch_max_bytes * 10), opaque)
self.dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection"
except cb_bin_client.MemcachedError:
return "error: DCP connect memcached error"
except socket.error:
return "error: DCP connection error"
try:
opaque = self.r.randint(0, 2**32)
self.dcp_conn._sendCmd(couchbaseConstants.CMD_DCP_CONTROL,
couchbaseConstants.KEY_DCP_EXT_METADATA,
bool_to_str(True), opaque)
self.dcp_conn._handleSingleResponse(opaque)
except EOFError:
return "error: Fail to set up DCP connection"
except cb_bin_client.MemcachedError:
pass
except socket.error:
return "error: DCP connection error"
self.running = True
self.start()
self.add_start_event(self.dcp_conn)
self.setup_dcp_streams()
return 0