def _handleKeyedResponse(self, myopaque):
response = ""
while len(response) < MIN_RECV_PACKET:
data = self.s.recv(MIN_RECV_PACKET - len(response))
if data == '':
raise exceptions.EOFError("Got empty data (remote died?).")
response += data
assert len(response) == MIN_RECV_PACKET
magic, cmd, keylen, extralen, dtype, errcode, remaining, opaque, cas=\
struct.unpack(RES_PKT_FMT, response)
rv = ""
while remaining > 0:
data = self.s.recv(remaining)
if data == '':
raise exceptions.EOFError("Got empty data (remote died?).")
rv += data
remaining -= len(data)
assert (magic
in (RES_MAGIC_BYTE, REQ_MAGIC_BYTE)), "Got magic: %d" % magic
assert myopaque is None or opaque == myopaque, \
"expected opaque %x, got %x" % (myopaque, opaque)
if errcode != 0:
raise MemcachedError(errcode, rv)
return cmd, opaque, cas, keylen, extralen, rv
def _recvMsg(self):
response = ""
while len(response) < MemcachedConstants.MIN_RECV_PACKET:
data = self.s.recv(MemcachedConstants.MIN_RECV_PACKET -
len(response))
if data == '':
raise exceptions.EOFError(
"Got empty data (remote died?). from {0}".format(
self.host))
response += data
assert len(response) == MemcachedConstants.MIN_RECV_PACKET
magic, cmd, keylen, extralen, dtype, errcode, remaining, opaque, cas =\
struct.unpack(MemcachedConstants.RES_PKT_FMT, response)
rv = ""
while remaining > 0:
data = self.s.recv(remaining)
if data == '':
raise exceptions.EOFError(
"Got empty data (remote died?). from {0}".format(
self.host))
rv += data
remaining -= len(data)
assert (magic in (
MemcachedConstants.RES_MAGIC_BYTE,
MemcachedConstants.REQ_MAGIC_BYTE)), "Got magic: %d" % magic
return cmd, errcode, opaque, cas, keylen, extralen, rv
def _recvMsg(self):
response = ""
while len(response) < MIN_RECV_PACKET:
r, _, _ = select.select([self.s], [], [], self.timeout)
if r:
data = self.s.recv(MIN_RECV_PACKET - len(response))
if data == '':
raise exceptions.EOFError("Got empty data (remote died?). from {0}".format(self.host))
response += data
else:
raise exceptions.EOFError("Timeout waiting for socket recv. from {0}".format(self.host))
assert len(response) == MIN_RECV_PACKET
magic, cmd, keylen, extralen, dtype, errcode, remaining, opaque, cas=\
struct.unpack(RES_PKT_FMT, response)
rv = ""
while remaining > 0:
r, _, _ = select.select([self.s], [], [], self.timeout)
if r:
data = self.s.recv(remaining)
if data == '':
raise exceptions.EOFError("Got empty data (remote died?). from {0}".format(self.host))
rv += data
remaining -= len(data)
else:
raise exceptions.EOFError("Timeout waiting for socket recv. from {0}".format(self.host))
assert (magic in (RES_MAGIC_BYTE, REQ_MAGIC_BYTE)), "Got magic: %d" % magic
return cmd, errcode, opaque, cas, keylen, extralen, rv
def _recvMsg(self):
response = ""
while len(response) < MIN_RECV_PACKET:
r, _, _ = select.select([self.s], [], [], self.timeout)
if r:
data = self.s.recv(MIN_RECV_PACKET - len(response))
if data == '':
raise exceptions.EOFError(
"Got empty data (remote died?). from {0}".format(
self.host))
response += data
else:
raise exceptions.EOFError(
"Timeout waiting for socket recv. from {0}".format(
self.host))
assert len(response) == MIN_RECV_PACKET
# Peek at the magic so we can support alternative-framing
magic = struct.unpack(">B", response[0:1])[0]
assert (magic in (RES_MAGIC_BYTE, REQ_MAGIC_BYTE, ALT_RES_MAGIC_BYTE,
ALT_REQ_MAGIC_BYTE)), "Got magic: 0x%x" % magic
cmd = 0
frameextralen = 0
keylen = 0
extralen = 0
dtype = 0
errcode = 0
remaining = 0
opaque = 0
cas = 0
if magic == ALT_RES_MAGIC_BYTE or magic == ALT_REQ_MAGIC_BYTE:
magic, cmd, frameextralen, keylen, extralen, dtype, errcode, remaining, opaque, cas = \
struct.unpack(ALT_RES_PKT_FMT, response)
else:
magic, cmd, keylen, extralen, dtype, errcode, remaining, opaque, cas = \
struct.unpack(RES_PKT_FMT, response)
rv = ""
while remaining > 0:
r, _, _ = select.select([self.s], [], [], self.timeout)
if r:
data = self.s.recv(remaining)
if data == '':
raise exceptions.EOFError(
"Got empty data (remote died?). from {0}".format(
self.host))
rv += data
remaining -= len(data)
else:
raise exceptions.EOFError(
"Timeout waiting for socket recv. from {0}".format(
self.host))
return cmd, errcode, opaque, cas, keylen, extralen, dtype, rv, frameextralen
def _recvData(self, length):
r, _, _ = select.select([self.s], [], [], self.timeout)
if r:
response = ""
while len(response) < length + 2:
data = self.s.recv((length + 2) - len(response))
if data == '':
raise exceptions.EOFError("Got empty data (remote died?). from {0}".format(self.host))
response += data
return response[:-2]
else:
raise exceptions.EOFError("Timeout waiting for socket recv. from {0}".format(self.host))
def _recvMsg(self):
r, _, _ = select.select([self.s], [], [], self.timeout)
if r:
response = ""
while not response.endswith("\r\n"):
data = self.s.recv(1)
if data == '':
raise exceptions.EOFError("Got empty data (remote died?). from {0}".format(self.host))
response += data
return response[:-2]
else:
raise exceptions.EOFError("Timeout waiting for socket recv. from {0}".format(self.host))
def _sendMsg(self,
cmd,
key,
val,
opaque,
extraHeader='',
cas=0,
dtype=0,
vbucketId=0,
fmt=REQ_PKT_FMT,
magic=REQ_MAGIC_BYTE,
extended_meta_data='',
extraHeaderLength=None):
# a little bit unfortunate but the delWithMeta command expects the extra data length to be the
# overall packet length (28 in that case) so we need a facility to support that
if extraHeaderLength is None:
extraHeaderLength = len(extraHeader)
msg = struct.pack(
fmt, magic, cmd, len(key), extraHeaderLength, dtype, vbucketId,
len(key) + len(extraHeader) + len(val) + len(extended_meta_data),
opaque, cas)
_, w, _ = select.select([], [self.s], [], self.timeout)
if w:
self.s.send(msg + extraHeader + key + val + extended_meta_data)
else:
raise exceptions.EOFError(
"Timeout waiting for socket send. from {0}".format(self.host))
def get_bucket_type(self):
# get bucket type from web console
node_url = "http://{0}:8091/pools/default/buckets".format(self.host)
r = json.loads(urllib2.urlopen(node_url).read())
for k in r[0]:
if "bucketType" in k:
return r[0]["bucketType"]
else:
raise exceptions.EOFError("bucketType does not exist in node {0}".format(self.host))
def _sendMsg(self, cmd, key, val, opaque, extraHeader='', cas=0,
dtype=0, vbucketId=0,
fmt=REQ_PKT_FMT, magic=REQ_MAGIC_BYTE):
msg=struct.pack(fmt, magic,
cmd, len(key), len(extraHeader), dtype, vbucketId,
len(key) + len(extraHeader) + len(val), opaque, cas)
_, w, _ = select.select([], [self.s], [], self.timeout)
if w:
self.s.send(msg + extraHeader + key + val)
else:
raise exceptions.EOFError("Timeout waiting for socket send. from {0}".format(self.host))
def _recvMsg(self):
response = ""
while len(response) < MIN_RECV_PACKET:
data = self._socketRecv(MIN_RECV_PACKET - len(response))
if data == '':
raise exceptions.EOFError("Got empty data (remote died?).")
response += data
assert len(response) == MIN_RECV_PACKET
magic, cmd, keylen, extralen, dtype, errcode, remaining, opaque, cas=\
struct.unpack(RES_PKT_FMT, response)
rv = ""
while remaining > 0:
data = self._socketRecv(remaining)
if data == '':
raise exceptions.EOFError("Got empty data (remote died?).")
rv += data
remaining -= len(data)
assert (magic in (RES_MAGIC_BYTE, REQ_MAGIC_BYTE)), "Got magic: %d" % magic
return cmd, errcode, opaque, cas, keylen, extralen, rv
def run(self):
if not self.dcp_conn:
logging.error("socket to memcached server is not created yet.")
return
bytes_read = ''
rd_timeout = 1
desc = [self.dcp_conn.s]
extra_bytes = 0
while self.running:
try:
if self.dcp_done:
time.sleep(1)
continue
readers, writers, errors = select.select(
desc, [], [], rd_timeout)
rd_timeout = .25
if len(self.stream_list) == 0:
time.sleep(1)
continue
for reader in readers:
data = reader.recv(self.recv_min_bytes)
logging.debug("Read %d bytes off the wire" % len(data))
if len(data) == 0:
raise exceptions.EOFError(
"Got empty data (remote died?).")
bytes_read += data
while len(bytes_read) >= couchbaseConstants.MIN_RECV_PACKET:
magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas= \
struct.unpack(couchbaseConstants.RES_PKT_FMT, \
bytes_read[0:couchbaseConstants.MIN_RECV_PACKET])
if len(bytes_read) < (couchbaseConstants.MIN_RECV_PACKET +
bodylen):
extra_bytes = len(bytes_read)
break
rd_timeout = 0
body = bytes_read[couchbaseConstants.MIN_RECV_PACKET : \
couchbaseConstants.MIN_RECV_PACKET+bodylen]
bytes_read = bytes_read[couchbaseConstants.
MIN_RECV_PACKET + bodylen:]
self.response.put((opcode, status, opaque, cas, keylen, extlen, body, \
bodylen, datatype, \
couchbaseConstants.MIN_RECV_PACKET+bodylen+extra_bytes))
extra_bytes = 0
except socket.error:
break
except Exception, e:
pass
def _recv_dcp_msg(self, sock):
response = ""
while len(response) < couchbaseConstants.MIN_RECV_PACKET:
data = sock.recv(couchbaseConstants.MIN_RECV_PACKET - len(response))
if data == '':
raise exceptions.EOFError("Got empty data (remote died?).")
response += data
assert len(response) == couchbaseConstants.MIN_RECV_PACKET
magic, cmd, keylen, extralen, dtype, errcode, remaining, opaque, cas=\
struct.unpack(couchbaseConstants.RES_PKT_FMT, response)
datalen = remaining
rv = ""
while remaining > 0:
data = sock.recv(remaining)
if data == '':
raise exceptions.EOFError("Got empty data (remote died?).")
rv += data
remaining -= len(data)
assert (magic in (couchbaseConstants.RES_MAGIC_BYTE, couchbaseConstants.REQ_MAGIC_BYTE)), \
"Got magic: %d" % magic
return cmd, errcode, opaque, cas, keylen, extralen, rv, datalen
def read_one_align_info_from_file_object(f):
id_line = f.readline()
if len(id_line) == 0:
raise exceptions.EOFError()
id_line = id_line.strip()
sharp, id_, score = id_line.split()
assert sharp == "#"
score = float(score)
align_line = f.readline().strip()
alignment = []
for link in align_line.split():
left, right = link.split("-")
left = [int(x) for x in left.split(",")]
right = [int(x) for x in right.split(",")]
alignment.append((left, right))
return id_, score, alignment
def read_one_parse_info_from_file_object(f):
id_line = f.readline()
if len(id_line) == 0:
raise exceptions.EOFError()
sharp, id_part, score_part = id_line.split()
assert sharp == "#"
id_tag, id_ = id_part.split("=")
assert id_tag == "ID"
score_tag, score = score_part.split("=")
assert score_tag == "SCORE"
score = float(score)
sentence = []
while 1:
line = f.readline().strip()
if len(line) == 0:
return id_, sentence
splitted_line = line.split("\t")
num_pos = int(splitted_line[0])
dpnd = int(splitted_line[1])
word = splitted_line[2]
assert num_pos == len(sentence)
sentence.append(word)
def _sendMsg(self, cmd):
_, w, _ = select.select([], [self.s], [], self.timeout)
if w:
self.s.send(cmd)
else:
raise exceptions.EOFError("Timeout waiting for socket send. from {0}".format(self.host))
def _socketRecv(self, amount):
ready = select.select([self.s], [], [], 30)
if ready[0]:
return self.s.recv(amount)
raise exceptions.EOFError("Operation timed out")