def total_msgs(opts, source_bucket, source_node, source_map):
source_name = source_node.get("hostname", None)
if not source_name:
return 0, None
spec = source_map["spec"]
name = source_bucket["name"]
vbuckets_num = len(source_bucket["vBucketServerMap"]["vBucketMap"])
if not vbuckets_num:
return 0, None
vbucket_list = getattr(opts, "vbucket_list", None)
stats_vals = {}
host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091)
for stats in ["curr_items", "vb_active_resident_items_ratio"]:
path = "/pools/default/buckets/%s/stats/%s" % (name, stats)
err, json, data = pump.rest_request_json(host, int(port), user, pswd, opts.ssl, path, reason="total_msgs")
if err:
return 0, None
nodeStats = data.get("nodeStats", None)
if not nodeStats:
return 0, None
vals = nodeStats.get(source_name, None)
if not vals:
return 0, None
stats_vals[stats] = vals[-1]
total_msgs = stats_vals["curr_items"]
resident_ratio = stats_vals["vb_active_resident_items_ratio"]
if 0 < resident_ratio < 100:
# for DGM case, server will transfer both in-memory items and backfill all items on disk
total_msgs += (resident_ratio / 100.0) * stats_vals["curr_items"]
return 0, int(total_msgs)
def provide_design(opts, source_spec: str, source_bucket: Dict[str, Any], source_map) -> \
Tuple[couchbaseConstants.PUMP_ERROR, Optional[str]]:
# Ephemeral buckets do not have design docs
if source_bucket['bucketType'] == 'ephemeral':
return 0, None
spec_parts = source_map.get('spec_parts')
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts)
if not source_nodes:
source_nodes = source_bucket['nodes']
if not source_nodes:
return f'error: no design source node; spec_parts: {spec_parts}', None
couch_api_base = source_nodes[0].get('couchApiBase')
if not couch_api_base:
return 0, None # No couchApiBase; probably not 2.0.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), user, pswd, opts.ssl,
"/pools/default/buckets/%s/ddocs" %
(source_bucket['name']),
reason="provide_design", verify=opts.no_ssl_verify, cacert=opts.cacert)
if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4.
ddocs_json = None
ddocs_url = f'{couch_api_base}/_all_docs'
ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true"
host, port, user, pswd, path = \
pump.parse_spec(opts, ddocs_url, 8092)
# Not using user/pwd as 2.0-DP4 CAPI did not support auth.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), None, None, opts.ssl,
path + ddocs_qry,
reason="provide_design-2.0DP4", verify=opts.no_ssl_verify, cacert=opts.cacert)
if err is not None:
return err, None
if ddocs is None:
return 0, None
if not ddocs.get('rows', None):
return 0, None
else:
return 0, json.dumps(ddocs.get('rows', []))
def provide_design(opts, source_spec: str, source_bucket: Dict[str, Any], source_map) -> \
Tuple[couchbaseConstants.PUMP_ERROR, Optional[str]]:
# Ephemeral buckets do not have design docs
if source_bucket['bucketType'] == 'ephemeral':
return 0, None
spec_parts = source_map.get('spec_parts')
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts)
if not source_nodes:
source_nodes = source_bucket['nodes']
if not source_nodes:
return f'error: no design source node; spec_parts: {spec_parts}', None
couch_api_base = source_nodes[0].get('couchApiBase')
if not couch_api_base:
return 0, None # No couchApiBase; probably not 2.0.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), user, pswd, opts.ssl,
"/pools/default/buckets/%s/ddocs" %
(source_bucket['name']),
reason="provide_design", verify=opts.no_ssl_verify, cacert=opts.cacert)
if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4.
ddocs_json = None
ddocs_url = f'{couch_api_base}/_all_docs'
ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true"
host, port, user, pswd, path = \
pump.parse_spec(opts, ddocs_url, 8092)
# Not using user/pwd as 2.0-DP4 CAPI did not support auth.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), None, None, opts.ssl,
path + ddocs_qry,
reason="provide_design-2.0DP4", verify=opts.no_ssl_verify, cacert=opts.cacert)
if err is not None:
return err, None
if ddocs is None:
return 0, None
if not ddocs.get('rows', None):
return 0, None
else:
return 0, json.dumps(ddocs.get('rows', []))
def provide_design(opts, source_spec, source_bucket, source_map):
spec_parts = source_map.get("spec_parts")
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts)
if not source_nodes:
source_nodes = source_bucket["nodes"]
if not source_nodes:
return ("error: no design source node; spec_parts: %s" % (spec_parts,), None)
couch_api_base = source_nodes[0].get("couchApiBase")
if not couch_api_base:
return 0, None # No couchApiBase; probably not 2.0.
err, ddocs_json, ddocs = pump.rest_request_json(
host,
int(port),
user,
pswd,
opts.ssl,
"/pools/default/buckets/%s/ddocs" % (source_bucket["name"]),
reason="provide_design",
)
if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4.
ddocs_json = None
ddocs_url = couch_api_base + "/_all_docs"
ddocs_qry = '?startkey="_design/"&endkey="_design0"&include_docs=true'
host, port, user, pswd, path = pump.parse_spec(opts, ddocs_url, 8092)
# Not using user/pwd as 2.0-DP4 CAPI did not support auth.
err, ddocs_json, ddocs = pump.rest_request_json(
host, int(port), None, None, opts.ssl, path + ddocs_qry, reason="provide_design-2.0DP4"
)
if err:
return err, None
if not ddocs.get("rows", None):
return 0, None
else:
return 0, json.dumps(ddocs.get("rows", []))
def provide_design(opts, source_spec, source_bucket, source_map):
spec_parts = source_map.get('spec_parts')
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts)
if not source_nodes:
source_nodes = source_bucket['nodes']
if not source_nodes:
return ("error: no design source node; spec_parts: %s" %
(spec_parts, ), None)
couch_api_base = source_nodes[0].get('couchApiBase')
if not couch_api_base:
return 0, None # No couchApiBase; probably not 2.0.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), user, pswd, opts.ssl,
"/pools/default/buckets/%s/ddocs" %
(source_bucket['name']),
reason="provide_design")
if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4.
ddocs_json = None
ddocs_url = couch_api_base + "/_all_docs"
ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true"
host, port, user, pswd, path = \
pump.parse_spec(opts, ddocs_url, 8092)
# Not using user/pwd as 2.0-DP4 CAPI did not support auth.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), None, None, opts.ssl,
path + ddocs_qry,
reason="provide_design-2.0DP4")
if err:
return err, None
if not ddocs.get('rows', None):
return 0, None
else:
return 0, json.dumps(ddocs.get('rows', []))
def provide_design(opts, source_spec, source_bucket, source_map):
spec_parts = source_map.get('spec_parts')
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts)
if not source_nodes:
if spec_parts[0] not in ['localhost', '127.0.0.1']:
return ("error: no design source node; spec_parts: %s" %
(spec_parts,), None)
else:
source_nodes = source_bucket['nodes']
couch_api_base = source_nodes[0].get('couchApiBase')
if not couch_api_base:
return 0, None # No couchApiBase; probably not 2.0.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), user, pswd,
"/pools/default/buckets/%s/ddocs" %
(source_bucket['name']),
reason="provide_design")
if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4.
ddocs_json = None
ddocs_url = couch_api_base + "/_all_docs"
ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true"
host, port, user, pswd, path = \
pump.parse_spec(opts, ddocs_url, 8092)
# Not using user/pwd as 2.0-DP4 CAPI did not support auth.
err, ddocs_json, ddocs = \
pump.rest_request_json(host, int(port), None, None,
path + ddocs_qry,
reason="provide_design-2.0DP4")
if err:
return err, None
return 0, json.dumps(ddocs.get('rows', []))
def total_msgs(opts, source_bucket: Dict[str, Any], source_node, source_map: Dict[str, Any]) ->\
Tuple[couchbaseConstants.PUMP_ERROR, Optional[int]]:
source_name = source_node.get("hostname", None)
if not source_name:
return 0, None
spec = source_map['spec']
name = source_bucket['name']
vbuckets_num = len(source_bucket['vBucketServerMap']['vBucketMap'])
if not vbuckets_num:
return 0, None
vbucket_list = getattr(opts, "vbucket_list", None)
stats_vals = {}
host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091)
for stats in ["curr_items", "vb_active_resident_items_ratio"]:
path = f'/pools/default/buckets/{name}/stats/{stats}'
err, json, data = pump.rest_request_json(host,
int(port),
user,
pswd,
opts.ssl,
path,
reason="total_msgs",
verify=opts.no_ssl_verify,
cacert=opts.cacert)
if err:
return 0, None
if data is not None:
nodeStats = data.get("nodeStats", None)
if not nodeStats:
return 0, None
vals = nodeStats.get(source_name, None)
if not vals:
return 0, None
stats_vals[stats] = vals[-1]
total_msgs = stats_vals["curr_items"]
resident_ratio = stats_vals["vb_active_resident_items_ratio"]
if 0 < resident_ratio < 100:
# for DGM case, server will transfer both in-memory items and
# backfill all items on disk
total_msgs += (resident_ratio / 100.0) * stats_vals["curr_items"]
return 0, int(total_msgs)
def provide_index(opts, source_spec, source_bucket, source_map):
err, index_server = pump.filter_server(opts, source_spec, 'index')
if err or not index_server:
logging.warning("could not find index server:%s" % err)
return 0, None
spec_parts = source_map.get('spec_parts')
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
host, port = pump.hostport(index_server)
err, ddocs_json, ddocs = \
pump.rest_request_json(host, couchbaseConstants.INDEX_PORT, user, pswd, opts.ssl,
"/getIndexMetadata?bucket=%s" % (source_bucket['name']),
reason="provide_index")
if err:
return err, None
return 0, json.dumps(ddocs["result"])
def provide_index(opts, source_spec, source_bucket, source_map):
err, index_server = pump.filter_server(opts, source_spec, 'index')
if err or not index_server:
logging.warning("could not find index server:%s" % err)
return 0, None
spec_parts = source_map.get('spec_parts')
if not spec_parts:
return "error: no design spec_parts", None
host, port, user, pswd, path = spec_parts
host, port = pump.hostport(index_server)
err, ddocs_json, ddocs = \
pump.rest_request_json(host, couchbaseConstants.INDEX_PORT, user, pswd, opts.ssl,
"/getIndexMetadata?bucket=%s" % (source_bucket['name']),
reason="provide_index")
if err:
return err, None
return 0, json.dumps(ddocs["result"])
def total_msgs(opts, source_bucket, source_node, source_map):
source_name = source_node.get("hostname", None)
if not source_name:
return 0, None
spec = source_map['spec']
name = source_bucket['name']
vbuckets_num = len(source_bucket['vBucketServerMap']['vBucketMap'])
if not vbuckets_num:
return 0, None
vbucket_list = getattr(opts, "vbucket_list", None)
stats_vals = {}
host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091)
for stats in ["curr_items", "vb_active_resident_items_ratio"]:
path = "/pools/default/buckets/%s/stats/%s" % (name, stats)
err, json, data = pump.rest_request_json(host,
int(port),
user,
pswd,
opts.ssl,
path,
reason="total_msgs")
if err:
return 0, None
nodeStats = data.get("nodeStats", None)
if not nodeStats:
return 0, None
vals = nodeStats.get(source_name, None)
if not vals:
return 0, None
stats_vals[stats] = vals[-1]
total_msgs = stats_vals["curr_items"]
resident_ratio = stats_vals["vb_active_resident_items_ratio"]
if 0 < resident_ratio < 100:
#for DGM case, server will transfer both in-memory items and
#backfill all items on disk
total_msgs += (resident_ratio / 100.0) * stats_vals["curr_items"]
return 0, int(total_msgs)
def total_msgs(opts, source_bucket: Dict[str, Any], source_node, source_map: Dict[str, Any]) ->\
Tuple[couchbaseConstants.PUMP_ERROR, Optional[int]]:
source_name = source_node.get("hostname", None)
if not source_name:
return 0, None
spec = source_map['spec']
name = source_bucket['name']
vbuckets_num = len(source_bucket['vBucketServerMap']['vBucketMap'])
if not vbuckets_num:
return 0, None
vbucket_list = getattr(opts, "vbucket_list", None)
stats_vals = {}
host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091)
for stats in ["curr_items", "vb_active_resident_items_ratio"]:
path = f'/pools/default/buckets/{name}/stats/{stats}'
err, json, data = pump.rest_request_json(host, int(port),
user, pswd, opts.ssl, path,
reason="total_msgs", verify=opts.no_ssl_verify, cacert=opts.cacert)
if err:
return 0, None
if data is not None:
nodeStats = data.get("nodeStats", None)
if not nodeStats:
return 0, None
vals = nodeStats.get(source_name, None)
if not vals:
return 0, None
stats_vals[stats] = vals[-1]
total_msgs = stats_vals["curr_items"]
resident_ratio = stats_vals["vb_active_resident_items_ratio"]
if 0 < resident_ratio < 100:
# for DGM case, server will transfer both in-memory items and
# backfill all items on disk
total_msgs += (resident_ratio/100.0) * stats_vals["curr_items"]
return 0, int(total_msgs)
def total_msgs(opts, source_bucket, source_node, source_map):
source_name = source_node.get("hostname", None)
if not source_name:
return 0, None
spec = source_map["spec"]
name = source_bucket["name"]
path = "/pools/default/buckets/%s/stats/curr_items" % (name)
host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091)
err, json, data = pump.rest_request_json(host, int(port), user, pswd, path, reason="total_msgs")
if err:
return 0, None
nodeStats = data.get("nodeStats", None)
if not nodeStats:
return 0, None
curr_items = nodeStats.get(source_name, None)
if not curr_items:
return 0, None
return 0, curr_items[-1]
def encode_tap_connect_opts(opts, backfill=False, vblist=None):
header = 0
val = []
for op in sorted(opts.keys()):
header |= op
if op in memcacheConstants.TAP_FLAG_TYPES:
val.append(struct.pack(memcacheConstants.TAP_FLAG_TYPES[op],
opts[op]))
elif backfill and op == memcacheConstants.TAP_FLAG_CHECKPOINT:
if opts[op][2] >= 0:
val.append(struct.pack(">HHQ",
opts[op][0], opts[op][1], opts[op][2]))
elif vblist and op == memcacheConstants.TAP_FLAG_LIST_VBUCKETS:
val.apend(struct.pack(">H", len(vblist))
vblist = vblist[1:-1].split(",")
for v in vblist:
val.apend(struct.pack(">H", int(v)))
else:
val.append(opts[op])
return struct.pack(">I", header), ''.join(val)
@staticmethod
def total_msgs(opts, source_bucket, source_node, source_map):
source_name = source_node.get("hostname", None)
if not source_name:
return 0, None
spec = source_map['spec']
name = source_bucket['name']
path = "/pools/default/buckets/%s/stats/curr_items" % (name)
host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091)
err, json, data = pump.rest_request_json(host, int(port),
user, pswd, path,
reason="total_msgs")
if err:
return 0, None
nodeStats = data.get("nodeStats", None)
if not nodeStats:
return 0, None
curr_items = nodeStats.get(source_name, None)
if not curr_items:
return 0, None
return 0, curr_items[-1]
class TapSink(pump_mc.CBSink):
"""Smart client sink using tap protocal to couchbase cluster."""
def __init__(self, opts, spec, source_bucket, source_node,
source_map, sink_map, ctl, cur):
super(TapSink, self).__init__(opts, spec, source_bucket, source_node,
source_map, sink_map, ctl, cur)
self.tap_name = "".join(random.sample(string.letters, 16))
@staticmethod
def check_base(opts, spec):
#allow destination vbucket state to be anything
op = getattr(opts, "destination_operation", None)
if not op in [None, 'set', 'add', 'get']:
return ("error: --destination-operation unsupported value: %s" +
"; use set, add, get") % (op)
return pump.EndPoint.check_base(opts, spec)
def find_conn(self, mconns, vbucket_id):
rc, conn = super(TapSink, self).find_conn(mconns, vbucket_id)
if rc != 0:
return rc, None
tap_opts = {memcacheConstants.TAP_FLAG_SUPPORT_ACK: ''}
conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"]
if self.tap_conn.tap_fix_flag_byteorder:
tap_opts[memcacheConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = ''
ext, val = TapSink.encode_tap_connect_opts(tap_opts)
conn._sendCmd(memcacheConstants.CMD_TAP_CONNECT,
self.tap_name, val, 0, ext)
return rv, conn
def send_msgs(self, conn, msgs, operation, vbucket_id=None):
rv = super(TapSink, self).sendMsg(conn, msgs, operation, vbucket_id)
if rv != 0:
return rv
#send vbucket recovery commit msg
host, port, user, pwd, path = \
pump.parse_spec(self.opts, self.sink, 8091)
params={"vbucket", vbucket_id}
err, conn = \
pump.rest_request(host, int(port), user, pwd,
'/pools/default/buckets/%s/commitVbucketRecovery' % self.sink_bucket,
method='POST', body=params, reason='notify vbucket recovery done')
if err:
logging.error("error: fail to notify that vbucket msg transferring is done")
return rv
