def total_msgs(opts, source_bucket, source_node, source_map): source_name = source_node.get("hostname", None) if not source_name: return 0, None spec = source_map["spec"] name = source_bucket["name"] vbuckets_num = len(source_bucket["vBucketServerMap"]["vBucketMap"]) if not vbuckets_num: return 0, None vbucket_list = getattr(opts, "vbucket_list", None) stats_vals = {} host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091) for stats in ["curr_items", "vb_active_resident_items_ratio"]: path = "/pools/default/buckets/%s/stats/%s" % (name, stats) err, json, data = pump.rest_request_json(host, int(port), user, pswd, opts.ssl, path, reason="total_msgs") if err: return 0, None nodeStats = data.get("nodeStats", None) if not nodeStats: return 0, None vals = nodeStats.get(source_name, None) if not vals: return 0, None stats_vals[stats] = vals[-1] total_msgs = stats_vals["curr_items"] resident_ratio = stats_vals["vb_active_resident_items_ratio"] if 0 < resident_ratio < 100: # for DGM case, server will transfer both in-memory items and backfill all items on disk total_msgs += (resident_ratio / 100.0) * stats_vals["curr_items"] return 0, int(total_msgs)
def provide_design(opts, source_spec: str, source_bucket: Dict[str, Any], source_map) -> \ Tuple[couchbaseConstants.PUMP_ERROR, Optional[str]]: # Ephemeral buckets do not have design docs if source_bucket['bucketType'] == 'ephemeral': return 0, None spec_parts = source_map.get('spec_parts') if not spec_parts: return "error: no design spec_parts", None host, port, user, pswd, path = spec_parts source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts) if not source_nodes: source_nodes = source_bucket['nodes'] if not source_nodes: return f'error: no design source node; spec_parts: {spec_parts}', None couch_api_base = source_nodes[0].get('couchApiBase') if not couch_api_base: return 0, None # No couchApiBase; probably not 2.0. err, ddocs_json, ddocs = \ pump.rest_request_json(host, int(port), user, pswd, opts.ssl, "/pools/default/buckets/%s/ddocs" % (source_bucket['name']), reason="provide_design", verify=opts.no_ssl_verify, cacert=opts.cacert) if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4. ddocs_json = None ddocs_url = f'{couch_api_base}/_all_docs' ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true" host, port, user, pswd, path = \ pump.parse_spec(opts, ddocs_url, 8092) # Not using user/pwd as 2.0-DP4 CAPI did not support auth. err, ddocs_json, ddocs = \ pump.rest_request_json(host, int(port), None, None, opts.ssl, path + ddocs_qry, reason="provide_design-2.0DP4", verify=opts.no_ssl_verify, cacert=opts.cacert) if err is not None: return err, None if ddocs is None: return 0, None if not ddocs.get('rows', None): return 0, None else: return 0, json.dumps(ddocs.get('rows', []))
def provide_design(opts, source_spec, source_bucket, source_map): spec_parts = source_map.get("spec_parts") if not spec_parts: return "error: no design spec_parts", None host, port, user, pswd, path = spec_parts source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts) if not source_nodes: source_nodes = source_bucket["nodes"] if not source_nodes: return ("error: no design source node; spec_parts: %s" % (spec_parts,), None) couch_api_base = source_nodes[0].get("couchApiBase") if not couch_api_base: return 0, None # No couchApiBase; probably not 2.0. err, ddocs_json, ddocs = pump.rest_request_json( host, int(port), user, pswd, opts.ssl, "/pools/default/buckets/%s/ddocs" % (source_bucket["name"]), reason="provide_design", ) if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4. ddocs_json = None ddocs_url = couch_api_base + "/_all_docs" ddocs_qry = '?startkey="_design/"&endkey="_design0"&include_docs=true' host, port, user, pswd, path = pump.parse_spec(opts, ddocs_url, 8092) # Not using user/pwd as 2.0-DP4 CAPI did not support auth. err, ddocs_json, ddocs = pump.rest_request_json( host, int(port), None, None, opts.ssl, path + ddocs_qry, reason="provide_design-2.0DP4" ) if err: return err, None if not ddocs.get("rows", None): return 0, None else: return 0, json.dumps(ddocs.get("rows", []))
def provide_design(opts, source_spec, source_bucket, source_map): spec_parts = source_map.get('spec_parts') if not spec_parts: return "error: no design spec_parts", None host, port, user, pswd, path = spec_parts source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts) if not source_nodes: source_nodes = source_bucket['nodes'] if not source_nodes: return ("error: no design source node; spec_parts: %s" % (spec_parts, ), None) couch_api_base = source_nodes[0].get('couchApiBase') if not couch_api_base: return 0, None # No couchApiBase; probably not 2.0. err, ddocs_json, ddocs = \ pump.rest_request_json(host, int(port), user, pswd, opts.ssl, "/pools/default/buckets/%s/ddocs" % (source_bucket['name']), reason="provide_design") if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4. ddocs_json = None ddocs_url = couch_api_base + "/_all_docs" ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true" host, port, user, pswd, path = \ pump.parse_spec(opts, ddocs_url, 8092) # Not using user/pwd as 2.0-DP4 CAPI did not support auth. err, ddocs_json, ddocs = \ pump.rest_request_json(host, int(port), None, None, opts.ssl, path + ddocs_qry, reason="provide_design-2.0DP4") if err: return err, None if not ddocs.get('rows', None): return 0, None else: return 0, json.dumps(ddocs.get('rows', []))
def provide_design(opts, source_spec, source_bucket, source_map): spec_parts = source_map.get('spec_parts') if not spec_parts: return "error: no design spec_parts", None host, port, user, pswd, path = spec_parts source_nodes = pump.filter_bucket_nodes(source_bucket, spec_parts) if not source_nodes: if spec_parts[0] not in ['localhost', '127.0.0.1']: return ("error: no design source node; spec_parts: %s" % (spec_parts,), None) else: source_nodes = source_bucket['nodes'] couch_api_base = source_nodes[0].get('couchApiBase') if not couch_api_base: return 0, None # No couchApiBase; probably not 2.0. err, ddocs_json, ddocs = \ pump.rest_request_json(host, int(port), user, pswd, "/pools/default/buckets/%s/ddocs" % (source_bucket['name']), reason="provide_design") if err and "response: 404" in err: # A 404/not-found likely means 2.0-DP4. ddocs_json = None ddocs_url = couch_api_base + "/_all_docs" ddocs_qry = "?startkey=\"_design/\"&endkey=\"_design0\"&include_docs=true" host, port, user, pswd, path = \ pump.parse_spec(opts, ddocs_url, 8092) # Not using user/pwd as 2.0-DP4 CAPI did not support auth. err, ddocs_json, ddocs = \ pump.rest_request_json(host, int(port), None, None, path + ddocs_qry, reason="provide_design-2.0DP4") if err: return err, None return 0, json.dumps(ddocs.get('rows', []))
def total_msgs(opts, source_bucket: Dict[str, Any], source_node, source_map: Dict[str, Any]) ->\ Tuple[couchbaseConstants.PUMP_ERROR, Optional[int]]: source_name = source_node.get("hostname", None) if not source_name: return 0, None spec = source_map['spec'] name = source_bucket['name'] vbuckets_num = len(source_bucket['vBucketServerMap']['vBucketMap']) if not vbuckets_num: return 0, None vbucket_list = getattr(opts, "vbucket_list", None) stats_vals = {} host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091) for stats in ["curr_items", "vb_active_resident_items_ratio"]: path = f'/pools/default/buckets/{name}/stats/{stats}' err, json, data = pump.rest_request_json(host, int(port), user, pswd, opts.ssl, path, reason="total_msgs", verify=opts.no_ssl_verify, cacert=opts.cacert) if err: return 0, None if data is not None: nodeStats = data.get("nodeStats", None) if not nodeStats: return 0, None vals = nodeStats.get(source_name, None) if not vals: return 0, None stats_vals[stats] = vals[-1] total_msgs = stats_vals["curr_items"] resident_ratio = stats_vals["vb_active_resident_items_ratio"] if 0 < resident_ratio < 100: # for DGM case, server will transfer both in-memory items and # backfill all items on disk total_msgs += (resident_ratio / 100.0) * stats_vals["curr_items"] return 0, int(total_msgs)
def provide_index(opts, source_spec, source_bucket, source_map): err, index_server = pump.filter_server(opts, source_spec, 'index') if err or not index_server: logging.warning("could not find index server:%s" % err) return 0, None spec_parts = source_map.get('spec_parts') if not spec_parts: return "error: no design spec_parts", None host, port, user, pswd, path = spec_parts host, port = pump.hostport(index_server) err, ddocs_json, ddocs = \ pump.rest_request_json(host, couchbaseConstants.INDEX_PORT, user, pswd, opts.ssl, "/getIndexMetadata?bucket=%s" % (source_bucket['name']), reason="provide_index") if err: return err, None return 0, json.dumps(ddocs["result"])
def total_msgs(opts, source_bucket, source_node, source_map): source_name = source_node.get("hostname", None) if not source_name: return 0, None spec = source_map['spec'] name = source_bucket['name'] vbuckets_num = len(source_bucket['vBucketServerMap']['vBucketMap']) if not vbuckets_num: return 0, None vbucket_list = getattr(opts, "vbucket_list", None) stats_vals = {} host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091) for stats in ["curr_items", "vb_active_resident_items_ratio"]: path = "/pools/default/buckets/%s/stats/%s" % (name, stats) err, json, data = pump.rest_request_json(host, int(port), user, pswd, opts.ssl, path, reason="total_msgs") if err: return 0, None nodeStats = data.get("nodeStats", None) if not nodeStats: return 0, None vals = nodeStats.get(source_name, None) if not vals: return 0, None stats_vals[stats] = vals[-1] total_msgs = stats_vals["curr_items"] resident_ratio = stats_vals["vb_active_resident_items_ratio"] if 0 < resident_ratio < 100: #for DGM case, server will transfer both in-memory items and #backfill all items on disk total_msgs += (resident_ratio / 100.0) * stats_vals["curr_items"] return 0, int(total_msgs)
def total_msgs(opts, source_bucket: Dict[str, Any], source_node, source_map: Dict[str, Any]) ->\ Tuple[couchbaseConstants.PUMP_ERROR, Optional[int]]: source_name = source_node.get("hostname", None) if not source_name: return 0, None spec = source_map['spec'] name = source_bucket['name'] vbuckets_num = len(source_bucket['vBucketServerMap']['vBucketMap']) if not vbuckets_num: return 0, None vbucket_list = getattr(opts, "vbucket_list", None) stats_vals = {} host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091) for stats in ["curr_items", "vb_active_resident_items_ratio"]: path = f'/pools/default/buckets/{name}/stats/{stats}' err, json, data = pump.rest_request_json(host, int(port), user, pswd, opts.ssl, path, reason="total_msgs", verify=opts.no_ssl_verify, cacert=opts.cacert) if err: return 0, None if data is not None: nodeStats = data.get("nodeStats", None) if not nodeStats: return 0, None vals = nodeStats.get(source_name, None) if not vals: return 0, None stats_vals[stats] = vals[-1] total_msgs = stats_vals["curr_items"] resident_ratio = stats_vals["vb_active_resident_items_ratio"] if 0 < resident_ratio < 100: # for DGM case, server will transfer both in-memory items and # backfill all items on disk total_msgs += (resident_ratio/100.0) * stats_vals["curr_items"] return 0, int(total_msgs)
def total_msgs(opts, source_bucket, source_node, source_map): source_name = source_node.get("hostname", None) if not source_name: return 0, None spec = source_map["spec"] name = source_bucket["name"] path = "/pools/default/buckets/%s/stats/curr_items" % (name) host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091) err, json, data = pump.rest_request_json(host, int(port), user, pswd, path, reason="total_msgs") if err: return 0, None nodeStats = data.get("nodeStats", None) if not nodeStats: return 0, None curr_items = nodeStats.get(source_name, None) if not curr_items: return 0, None return 0, curr_items[-1]
def encode_tap_connect_opts(opts, backfill=False, vblist=None): header = 0 val = [] for op in sorted(opts.keys()): header |= op if op in memcacheConstants.TAP_FLAG_TYPES: val.append(struct.pack(memcacheConstants.TAP_FLAG_TYPES[op], opts[op])) elif backfill and op == memcacheConstants.TAP_FLAG_CHECKPOINT: if opts[op][2] >= 0: val.append(struct.pack(">HHQ", opts[op][0], opts[op][1], opts[op][2])) elif vblist and op == memcacheConstants.TAP_FLAG_LIST_VBUCKETS: val.apend(struct.pack(">H", len(vblist)) vblist = vblist[1:-1].split(",") for v in vblist: val.apend(struct.pack(">H", int(v))) else: val.append(opts[op]) return struct.pack(">I", header), ''.join(val) @staticmethod def total_msgs(opts, source_bucket, source_node, source_map): source_name = source_node.get("hostname", None) if not source_name: return 0, None spec = source_map['spec'] name = source_bucket['name'] path = "/pools/default/buckets/%s/stats/curr_items" % (name) host, port, user, pswd, _ = pump.parse_spec(opts, spec, 8091) err, json, data = pump.rest_request_json(host, int(port), user, pswd, path, reason="total_msgs") if err: return 0, None nodeStats = data.get("nodeStats", None) if not nodeStats: return 0, None curr_items = nodeStats.get(source_name, None) if not curr_items: return 0, None return 0, curr_items[-1] class TapSink(pump_mc.CBSink): """Smart client sink using tap protocal to couchbase cluster.""" def __init__(self, opts, spec, source_bucket, source_node, source_map, sink_map, ctl, cur): super(TapSink, self).__init__(opts, spec, source_bucket, source_node, source_map, sink_map, ctl, cur) self.tap_name = "".join(random.sample(string.letters, 16)) @staticmethod def check_base(opts, spec): #allow destination vbucket state to be anything op = getattr(opts, "destination_operation", None) if not op in [None, 'set', 'add', 'get']: return ("error: --destination-operation unsupported value: %s" + "; use set, add, get") % (op) return pump.EndPoint.check_base(opts, spec) def find_conn(self, mconns, vbucket_id): rc, conn = super(TapSink, self).find_conn(mconns, vbucket_id) if rc != 0: return rc, None tap_opts = {memcacheConstants.TAP_FLAG_SUPPORT_ACK: ''} conn.tap_fix_flag_byteorder = version.split(".") >= ["2", "0", "0"] if self.tap_conn.tap_fix_flag_byteorder: tap_opts[memcacheConstants.TAP_FLAG_TAP_FIX_FLAG_BYTEORDER] = '' ext, val = TapSink.encode_tap_connect_opts(tap_opts) conn._sendCmd(memcacheConstants.CMD_TAP_CONNECT, self.tap_name, val, 0, ext) return rv, conn def send_msgs(self, conn, msgs, operation, vbucket_id=None): rv = super(TapSink, self).sendMsg(conn, msgs, operation, vbucket_id) if rv != 0: return rv #send vbucket recovery commit msg host, port, user, pwd, path = \ pump.parse_spec(self.opts, self.sink, 8091) params={"vbucket", vbucket_id} err, conn = \ pump.rest_request(host, int(port), user, pwd, '/pools/default/buckets/%s/commitVbucketRecovery' % self.sink_bucket, method='POST', body=params, reason='notify vbucket recovery done') if err: logging.error("error: fail to notify that vbucket msg transferring is done") return rv