def rest_request(host,
port,
user,
pswd,
ssl,
path,
method='GET',
body='',
reason='',
headers=None):
if reason:
reason = "; reason: %s" % (reason)
logging.debug("rest_request: %s@%s:%s%s%s" %
(tag_user_data(user), host, port, path, reason))
if ssl:
if port not in [
couchbaseConstants.SSL_REST_PORT,
couchbaseConstants.SSL_QUERY_PORT
]:
return ("error: invalid port %s used when ssl option is specified"
) % port, None, None
conn = httplib.HTTPSConnection(host, port)
else:
conn = httplib.HTTPConnection(host, port)
try:
header = rest_headers(user, pswd, headers)
conn.request(method, path, body, header)
resp = conn.getresponse()
except Exception, e:
return ("error: could not access REST API: %s:%s%s" +
"; please check source URL, server status, username (-u) and password (-p)" +
"; exception: %s%s") % \
(host, port, path, e, reason), None, None
def rest_request(host,
port,
user,
pswd,
use_ssl,
path,
method='GET',
body='',
reason='',
headers=None,
verify=True,
ca_cert=None):
if reason:
reason = "; reason: %s" % (reason)
logging.debug("rest_request: %s@%s:%s%s%s" %
(tag_user_data(user), host, port, path, reason))
if use_ssl:
if port not in [
couchbaseConstants.SSL_REST_PORT,
couchbaseConstants.SSL_QUERY_PORT
]:
return ("error: invalid port %s used when ssl option is specified"
) % port, None, None
ctx = ssl.create_default_context(cafile=ca_cert)
if not verify:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
conn = http.client.HTTPSConnection(host, port, context=ctx)
else:
conn = http.client.HTTPConnection(host, port)
try:
header = rest_headers(user, pswd, headers)
conn.request(method, path, body, header)
resp = conn.getresponse()
except Exception as e:
return ("error: could not access REST API: %s:%s%s" +
"; please check source URL, server status, username (-u) and password (-p)" +
"; exception: %s%s") % \
(host, port, path, e, reason), None, None
if resp.status in [200, 201, 202, 204, 302]:
return None, conn, resp.read()
conn.close()
if resp.status == 401:
return ("error: unable to access REST API: %s:%s%s" +
"; please check source URL, server status, username (-u) and password (-p)%s") % \
(host, port, path, reason), None, None
return ("error: unable to access REST API: %s:%s%s" +
"; please check source URL, server status, username (-u) and password (-p)" +
"; response: %s%s") % \
(host, port, path, resp.status, reason), None, None
def skip(self, key, vbucket_id):
if (self.only_key_re and not re.search(self.only_key_re, key)):
logging.warn("skipping msg with key: " + tag_user_data(key))
return True
if (self.only_vbucket_id is not None and
self.only_vbucket_id != vbucket_id):
logging.warn("skipping msg of vbucket_id: " + str(vbucket_id))
return True
return False
def skip(self, key, vbucket_id):
if (self.only_key_re and not re.search(self.only_key_re, key)):
logging.warn("skipping msg with key: " + tag_user_data(key))
return True
if (self.only_vbucket_id is not None and
self.only_vbucket_id != vbucket_id):
logging.warn("skipping msg of vbucket_id: " + str(vbucket_id))
return True
return False
def rest_request(host, port, user, pswd, ssl, path, method='GET', body='', reason='', headers=None):
if reason:
reason = "; reason: %s" % (reason)
logging.debug("rest_request: %s@%s:%s%s%s" % (tag_user_data(user), host, port, path, reason))
if ssl:
if port not in [couchbaseConstants.SSL_REST_PORT, couchbaseConstants.SSL_QUERY_PORT]:
return ("error: invalid port %s used when ssl option is specified") % port, None, None
conn = httplib.HTTPSConnection(host, port)
else:
conn = httplib.HTTPConnection(host, port)
try:
header = rest_headers(user, pswd, headers)
conn.request(method, path, body, header)
resp = conn.getresponse()
except Exception, e:
return ("error: could not access REST API: %s:%s%s" +
"; please check source URL, server status, username (-u) and password (-p)" +
"; exception: %s%s") % \
(host, port, path, e, reason), None, None
def recv_msgs(self, conn, msgs, vbucket_id=None, verify_opaque=True):
refresh = False
retry = False
for i, msg in enumerate(msgs):
cmd, vbucket_id_msg, key, flg, exp, cas, meta, val = msg[:8]
if vbucket_id is not None:
vbucket_id_msg = vbucket_id
if self.skip(key, vbucket_id_msg):
continue
try:
r_cmd, r_status, r_ext, r_key, r_val, r_cas, r_opaque = \
self.read_conn(conn)
if verify_opaque and i != r_opaque:
return "error: opaque mismatch: %s %s" % (i, r_opaque), None, None
if r_status == couchbaseConstants.ERR_SUCCESS:
continue
elif r_status == couchbaseConstants.ERR_KEY_EEXISTS:
#logging.warn("item exists: %s, key: %s" %
# (self.spec, tag_user_data(key)))
continue
elif r_status == couchbaseConstants.ERR_KEY_ENOENT:
if (cmd != couchbaseConstants.CMD_TAP_DELETE and
cmd != couchbaseConstants.CMD_GET):
logging.warn("item not found: %s, key: %s" %
(self.spec, tag_user_data(key)))
continue
elif (r_status == couchbaseConstants.ERR_ETMPFAIL or
r_status == couchbaseConstants.ERR_EBUSY or
r_status == couchbaseConstants.ERR_ENOMEM):
retry = True # Retry the whole batch again next time.
continue # But, finish recv'ing current batch.
elif r_status == couchbaseConstants.ERR_NOT_MY_VBUCKET:
msg = ("received NOT_MY_VBUCKET;"
" perhaps the cluster is/was rebalancing;"
" vbucket_id: %s, key: %s, spec: %s, host:port: %s:%s"
% (vbucket_id_msg, tag_user_data(key), self.spec,
conn.host, conn.port))
if self.opts.extra.get("nmv_retry", 1):
logging.warn("warning: " + msg)
refresh = True
retry = True
self.cur["tot_sink_not_my_vbucket"] = \
self.cur.get("tot_sink_not_my_vbucket", 0) + 1
else:
return "error: " + msg, None, None
elif r_status == couchbaseConstants.ERR_UNKNOWN_COMMAND:
if self.op_map == OP_MAP:
if not retry:
return "error: unknown command: %s" % (r_cmd), None, None
else:
if not retry:
logging.warn("destination does not take XXX-WITH-META"
" commands; will use META-less commands")
self.op_map = OP_MAP
retry = True
elif r_status == couchbaseConstants.ERR_ACCESS:
return json.loads(r_val)["error"]["context"], None, None
else:
return "error: MCSink MC error: " + str(r_status), None, None
except Exception, e:
logging.error("MCSink exception: %s", e)
return "error: MCSink exception: " + str(e), None, None