def rest_request(host, port, user, pswd, ssl, path, method='GET', body='', reason='', headers=None): if reason: reason = "; reason: %s" % (reason) logging.debug("rest_request: %s@%s:%s%s%s" % (tag_user_data(user), host, port, path, reason)) if ssl: if port not in [ couchbaseConstants.SSL_REST_PORT, couchbaseConstants.SSL_QUERY_PORT ]: return ("error: invalid port %s used when ssl option is specified" ) % port, None, None conn = httplib.HTTPSConnection(host, port) else: conn = httplib.HTTPConnection(host, port) try: header = rest_headers(user, pswd, headers) conn.request(method, path, body, header) resp = conn.getresponse() except Exception, e: return ("error: could not access REST API: %s:%s%s" + "; please check source URL, server status, username (-u) and password (-p)" + "; exception: %s%s") % \ (host, port, path, e, reason), None, None
def rest_request(host, port, user, pswd, use_ssl, path, method='GET', body='', reason='', headers=None, verify=True, ca_cert=None): if reason: reason = "; reason: %s" % (reason) logging.debug("rest_request: %s@%s:%s%s%s" % (tag_user_data(user), host, port, path, reason)) if use_ssl: if port not in [ couchbaseConstants.SSL_REST_PORT, couchbaseConstants.SSL_QUERY_PORT ]: return ("error: invalid port %s used when ssl option is specified" ) % port, None, None ctx = ssl.create_default_context(cafile=ca_cert) if not verify: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE conn = http.client.HTTPSConnection(host, port, context=ctx) else: conn = http.client.HTTPConnection(host, port) try: header = rest_headers(user, pswd, headers) conn.request(method, path, body, header) resp = conn.getresponse() except Exception as e: return ("error: could not access REST API: %s:%s%s" + "; please check source URL, server status, username (-u) and password (-p)" + "; exception: %s%s") % \ (host, port, path, e, reason), None, None if resp.status in [200, 201, 202, 204, 302]: return None, conn, resp.read() conn.close() if resp.status == 401: return ("error: unable to access REST API: %s:%s%s" + "; please check source URL, server status, username (-u) and password (-p)%s") % \ (host, port, path, reason), None, None return ("error: unable to access REST API: %s:%s%s" + "; please check source URL, server status, username (-u) and password (-p)" + "; response: %s%s") % \ (host, port, path, resp.status, reason), None, None
def skip(self, key, vbucket_id): if (self.only_key_re and not re.search(self.only_key_re, key)): logging.warn("skipping msg with key: " + tag_user_data(key)) return True if (self.only_vbucket_id is not None and self.only_vbucket_id != vbucket_id): logging.warn("skipping msg of vbucket_id: " + str(vbucket_id)) return True return False
def skip(self, key, vbucket_id): if (self.only_key_re and not re.search(self.only_key_re, key)): logging.warn("skipping msg with key: " + tag_user_data(key)) return True if (self.only_vbucket_id is not None and self.only_vbucket_id != vbucket_id): logging.warn("skipping msg of vbucket_id: " + str(vbucket_id)) return True return False
def rest_request(host, port, user, pswd, ssl, path, method='GET', body='', reason='', headers=None): if reason: reason = "; reason: %s" % (reason) logging.debug("rest_request: %s@%s:%s%s%s" % (tag_user_data(user), host, port, path, reason)) if ssl: if port not in [couchbaseConstants.SSL_REST_PORT, couchbaseConstants.SSL_QUERY_PORT]: return ("error: invalid port %s used when ssl option is specified") % port, None, None conn = httplib.HTTPSConnection(host, port) else: conn = httplib.HTTPConnection(host, port) try: header = rest_headers(user, pswd, headers) conn.request(method, path, body, header) resp = conn.getresponse() except Exception, e: return ("error: could not access REST API: %s:%s%s" + "; please check source URL, server status, username (-u) and password (-p)" + "; exception: %s%s") % \ (host, port, path, e, reason), None, None
def recv_msgs(self, conn, msgs, vbucket_id=None, verify_opaque=True): refresh = False retry = False for i, msg in enumerate(msgs): cmd, vbucket_id_msg, key, flg, exp, cas, meta, val = msg[:8] if vbucket_id is not None: vbucket_id_msg = vbucket_id if self.skip(key, vbucket_id_msg): continue try: r_cmd, r_status, r_ext, r_key, r_val, r_cas, r_opaque = \ self.read_conn(conn) if verify_opaque and i != r_opaque: return "error: opaque mismatch: %s %s" % (i, r_opaque), None, None if r_status == couchbaseConstants.ERR_SUCCESS: continue elif r_status == couchbaseConstants.ERR_KEY_EEXISTS: #logging.warn("item exists: %s, key: %s" % # (self.spec, tag_user_data(key))) continue elif r_status == couchbaseConstants.ERR_KEY_ENOENT: if (cmd != couchbaseConstants.CMD_TAP_DELETE and cmd != couchbaseConstants.CMD_GET): logging.warn("item not found: %s, key: %s" % (self.spec, tag_user_data(key))) continue elif (r_status == couchbaseConstants.ERR_ETMPFAIL or r_status == couchbaseConstants.ERR_EBUSY or r_status == couchbaseConstants.ERR_ENOMEM): retry = True # Retry the whole batch again next time. continue # But, finish recv'ing current batch. elif r_status == couchbaseConstants.ERR_NOT_MY_VBUCKET: msg = ("received NOT_MY_VBUCKET;" " perhaps the cluster is/was rebalancing;" " vbucket_id: %s, key: %s, spec: %s, host:port: %s:%s" % (vbucket_id_msg, tag_user_data(key), self.spec, conn.host, conn.port)) if self.opts.extra.get("nmv_retry", 1): logging.warn("warning: " + msg) refresh = True retry = True self.cur["tot_sink_not_my_vbucket"] = \ self.cur.get("tot_sink_not_my_vbucket", 0) + 1 else: return "error: " + msg, None, None elif r_status == couchbaseConstants.ERR_UNKNOWN_COMMAND: if self.op_map == OP_MAP: if not retry: return "error: unknown command: %s" % (r_cmd), None, None else: if not retry: logging.warn("destination does not take XXX-WITH-META" " commands; will use META-less commands") self.op_map = OP_MAP retry = True elif r_status == couchbaseConstants.ERR_ACCESS: return json.loads(r_val)["error"]["context"], None, None else: return "error: MCSink MC error: " + str(r_status), None, None except Exception, e: logging.error("MCSink exception: %s", e) return "error: MCSink exception: " + str(e), None, None