def consume_batch_async(self, batch):
op = self.operation()
op_mutate = op in ['set', 'add']
stdout = sys.stdout
msg_visitor = None
opts_etc = getattr(self.opts, "etc", None)
if opts_etc:
stdout = opts_etc.get("stdout", sys.stdout)
msg_visitor = opts_etc.get("msg_visitor", None)
mcd_compatible = self.opts.extra.get("mcd_compatible", 1)
msg_tuple_format = 0
for msg in batch.msgs:
if msg_visitor:
msg = msg_visitor(msg)
if not msg_tuple_format:
msg_tuple_format = len(msg)
cmd, vbucket_id, key, flg, exp, cas, meta, val = msg[:8]
seqno = dtype = nmeta = conf_res = 0
if msg_tuple_format > 8:
seqno, dtype, nmeta, conf_res = msg[8:]
if self.skip(key, vbucket_id):
continue
if dtype > 2:
try:
val = snappy.uncompress(val)
except Exception, err:
pass
try:
if cmd == couchbaseConstants.CMD_TAP_MUTATION or \
cmd == couchbaseConstants.CMD_DCP_MUTATION:
if op_mutate:
# <op> <key> <flags> <exptime> <bytes> [noreply]\r\n
if mcd_compatible:
stdout.write("%s %s %s %s %s\r\n" %
(op, key, flg, exp, len(val)))
else:
stdout.write("%s %s %s %s %s %s %s %s\r\n" %
(op, key, flg, exp, len(val), seqno,
dtype, conf_res))
stdout.write(val)
stdout.write("\r\n")
elif op == 'get':
stdout.write("get %s\r\n" % (key))
elif cmd == couchbaseConstants.CMD_TAP_DELETE or \
cmd == couchbaseConstants.CMD_DCP_DELETE:
if op_mutate:
stdout.write("delete %s\r\n" % (key))
elif cmd == couchbaseConstants.CMD_GET:
stdout.write("get %s\r\n" % (key))
else:
return "error: StdOutSink - unknown cmd: " + str(cmd), None
except IOError:
return "error: could not write to stdout", None
def consume_batch_async(self, batch):
op = self.operation()
op_mutate = op in ['set', 'add']
stdout = sys.stdout
msg_visitor = None
opts_etc = getattr(self.opts, "etc", None)
if opts_etc:
stdout = opts_etc.get("stdout", sys.stdout)
msg_visitor = opts_etc.get("msg_visitor", None)
mcd_compatible = self.opts.extra.get("mcd_compatible", 1)
msg_tuple_format = 0
for msg in batch.msgs:
if msg_visitor:
msg = msg_visitor(msg)
if not msg_tuple_format:
msg_tuple_format = len(msg)
cmd, vbucket_id, key, flg, exp, cas, meta, val = msg[:8]
seqno = dtype = nmeta = 0
if msg_tuple_format > 8:
seqno, dtype, nmeta = msg[8:]
if self.skip(key, vbucket_id):
continue
if dtype > 2:
try:
val = snappy.uncompress(val)
except Exception, err:
pass
try:
if cmd == couchbaseConstants.CMD_TAP_MUTATION or \
cmd == couchbaseConstants.CMD_DCP_MUTATION:
if op_mutate:
# <op> <key> <flags> <exptime> <bytes> [noreply]\r\n
if mcd_compatible:
stdout.write("%s %s %s %s %s\r\n" %
(op, key, flg, exp, len(val)))
else:
stdout.write("%s %s %s %s %s %s %s\r\n" %
(op, key, flg, exp, len(val), seqno, dtype))
stdout.write(val)
stdout.write("\r\n")
elif op == 'get':
stdout.write("get %s\r\n" % (key))
elif cmd == couchbaseConstants.CMD_TAP_DELETE or \
cmd == couchbaseConstants.CMD_DCP_DELETE:
if op_mutate:
stdout.write("delete %s\r\n" % (key))
elif cmd == couchbaseConstants.CMD_GET:
stdout.write("get %s\r\n" % (key))
else:
return "error: StdOutSink - unknown cmd: " + str(cmd), None
except IOError:
return "error: could not write to stdout", None
def send_msgs(self, conn, msgs, operation, vbucket_id=None):
m = []
msg_format_length = 0
for i, msg in enumerate(msgs):
if not msg_format_length:
msg_format_length = len(msg)
cmd, vbucket_id_msg, key, flg, exp, cas, meta, val = msg[:8]
seqno = dtype = nmeta = conf_res = 0
if msg_format_length > 8:
seqno, dtype, nmeta, conf_res = msg[8:]
if vbucket_id is not None:
vbucket_id_msg = vbucket_id
if self.skip(key, vbucket_id_msg):
continue
rv, cmd = self.translate_cmd(cmd, operation, meta)
if rv != 0:
return rv
if dtype > 2:
if self.uncompress and val:
try:
val = snappy.uncompress(val)
except Exception, err:
pass
if cmd == couchbaseConstants.CMD_GET:
val, flg, exp, cas = '', 0, 0, 0
if cmd == couchbaseConstants.CMD_NOOP:
key, val, flg, exp, cas = '', '', 0, 0, 0
if cmd == couchbaseConstants.CMD_DELETE:
val = ''
# A tombstone can contain Xattrs
if cmd == couchbaseConstants.CMD_DELETE_WITH_META and not dtype & couchbaseConstants.DATATYPE_HAS_XATTR:
val = ''
rv, req = self.cmd_request(cmd, vbucket_id_msg, key, val,
ctypes.c_uint32(flg).value, exp, cas,
meta, i, dtype, nmeta, conf_res)
if rv != 0:
return rv
self.append_req(m, req)
def send_msgs(self, conn, msgs, operation, vbucket_id=None):
m = []
msg_format_length = 0
for i, msg in enumerate(msgs):
if not msg_format_length:
msg_format_length = len(msg)
cmd, vbucket_id_msg, key, flg, exp, cas, meta, val = msg[:8]
seqno = dtype = nmeta = conf_res = 0
if msg_format_length > 8:
seqno, dtype, nmeta, conf_res = msg[8:]
if vbucket_id is not None:
vbucket_id_msg = vbucket_id
if self.skip(key, vbucket_id_msg):
continue
rv, cmd = self.translate_cmd(cmd, operation, meta)
if rv != 0:
return rv
if dtype > 2:
if self.uncompress and val:
try:
val = snappy.uncompress(val)
except Exception, err:
pass
if cmd == couchbaseConstants.CMD_GET:
val, flg, exp, cas = '', 0, 0, 0
if cmd == couchbaseConstants.CMD_NOOP:
key, val, flg, exp, cas = '', '', 0, 0, 0
if cmd in (couchbaseConstants.CMD_DELETE, couchbaseConstants.CMD_DELETE_WITH_META):
val = ''
rv, req = self.cmd_request(cmd, vbucket_id_msg, key, val,
ctypes.c_uint32(flg).value,
exp, cas, meta, i, dtype, nmeta,
conf_res)
if rv != 0:
return rv
self.append_req(m, req)
'id', 'flags', 'expiration', 'cas', 'value', 'rev', 'vbid',
'dtype'
])
msg_tuple_format = 0
for msg in batch.msgs:
cmd, vbucket_id, key, flg, exp, cas, meta, val = msg[:8]
if self.skip(key, vbucket_id):
continue
if not msg_tuple_format:
msg_tuple_format = len(msg)
seqno = dtype = nmeta = 0
if msg_tuple_format > 8:
seqno, dtype, nmeta, conf_res = msg[8:]
if dtype > 2:
try:
val = snappy.uncompress(val)
except Exception, err:
pass
try:
if cmd in [
couchbaseConstants.CMD_TAP_MUTATION,
couchbaseConstants.CMD_DCP_MUTATION
]:
if self.fields:
if val and len(val) > 0:
try:
row = []
doc = json.loads(val)
if type(doc) == dict:
for field in self.fields:
if field == 'id':
filename), None
self.writer = csv.writer(csvfile)
self.writer.writerow(['id', 'flags', 'expiration', 'cas', 'value', 'rev', 'vbid', 'dtype'])
msg_tuple_format = 0
for msg in batch.msgs:
cmd, vbucket_id, key, flg, exp, cas, meta, val = msg[:8]
if self.skip(key, vbucket_id):
continue
if not msg_tuple_format:
msg_tuple_format = len(msg)
seqno = dtype = nmeta = 0
if msg_tuple_format > 8:
seqno, dtype, nmeta, conf_res = msg[8:]
if dtype > 2:
try:
val = snappy.uncompress(val)
except Exception, err:
pass
try:
if cmd in [couchbaseConstants.CMD_TAP_MUTATION,
couchbaseConstants.CMD_DCP_MUTATION]:
if self.fields:
if val and len(val) > 0:
try:
row = []
doc = json.loads(val)
if type(doc) == dict:
for field in self.fields:
if field == 'id':
row.append(key)
else:
