def __init__(self, client, table, rows=None, columns=None):
"""
Construct a new TsObject.
:param client: A RiakClient object.
:type client: :class:`RiakClient <riak.client.RiakClient>`
:param table: The table for the timeseries data as a Table object.
:type table: :class:`Table` <riak.table.Table>
:param rows: An list of lists with timeseries data
:type rows: list
:param columns: A TsColumns tuple. Optional
:type columns: :class:`TsColumns` <riak.TsColumns>
"""
if not isinstance(table, Table):
raise ValueError('table must be an instance of Table.')
self.client = client
self.table = table
if rows is not None and not isinstance(rows, list):
raise RiakError("TsObject rows parameter must be a list.")
else:
self.rows = rows
if columns is not None and \
not isinstance(columns, TsColumns):
raise RiakError(
"TsObject columns parameter must be a TsColumns instance")
else:
self.columns = columns
def maybe_riak_error(self, msg_code, data=None):
if msg_code == riak.pb.messages.MSG_CODE_ERROR_RESP:
if data is None:
raise RiakError('no error provided!')
else:
err = parse_pbuf_msg(msg_code, data)
raise RiakError(bytes_to_str(err.errmsg))
def _recv_pkt(self):
nmsglen = self._socket.recv(4)
while len(nmsglen) < 4:
x = self._socket.recv(4 - len(nmsglen))
if not x:
break
nmsglen += x
if len(nmsglen) != 4:
raise RiakError(
"Socket returned short packet length %d - expected 4" %
len(nmsglen))
msglen, = struct.unpack('!i', nmsglen)
self._inbuf_len = msglen
if PY2:
self._inbuf = ''
else:
self._inbuf = bytes()
while len(self._inbuf) < msglen:
want_len = min(8192, msglen - len(self._inbuf))
recv_buf = self._socket.recv(want_len)
if not recv_buf:
break
self._inbuf += recv_buf
if len(self._inbuf) != self._inbuf_len:
raise RiakError("Socket returned short packet %d - expected %d" %
(len(self._inbuf), self._inbuf_len))
def __init__(self, client, table, rows=[], columns=[]):
"""
Construct a new TsObject.
:param client: A RiakClient object.
:type client: :class:`RiakClient <riak.client.RiakClient>`
:param table: The table for the timeseries data as a Table object.
:type table: :class:`Table` <riak.table.Table>
:param rows: An list of lists with timeseries data
:type rows: list
:param columns: An list of Column names and types. Optional.
:type columns: list
"""
if not isinstance(table, Table):
raise ValueError('table must be an instance of Table.')
self.client = client
self.table = table
self.rows = rows
if not isinstance(self.rows, list):
raise RiakError("TsObject requires a list of rows")
self.columns = columns
if self.columns is not None and not isinstance(self.columns, list):
raise RiakError("TsObject columns must be a list")
def _recv_msg(self, expect=None):
self._recv_pkt()
msg_code, = struct.unpack("B", self._inbuf[:1])
if msg_code is MSG_CODE_ERROR_RESP:
err = self._parse_msg(msg_code, self._inbuf[1:])
raise RiakError(err.errmsg)
elif msg_code in MESSAGE_CLASSES:
msg = self._parse_msg(msg_code, self._inbuf[1:])
else:
raise Exception("unknown msg code %s" % msg_code)
if expect and msg_code != expect:
raise RiakError("unexpected protocol buffer message code: %d, %r" %
(msg_code, msg))
return msg_code, msg
def get(self, robj, r=None, pr=None, vtag=None):
"""
Serialize get request and deserialize response
"""
if vtag is not None:
raise RiakError("PB transport does not support vtags")
bucket = robj.get_bucket()
req = riakclient_pb2.RpbGetReq()
req.r = self.translate_rw_val(r)
req.pr = self.translate_rw_val(pr)
req.bucket = bucket.get_name()
req.key = robj.get_key()
# An expected response code of None implies "any response is valid".
msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None)
if msg_code == MSG_CODE_GET_RESP:
contents = []
for c in resp.content:
contents.append(self.decode_content(c))
return resp.vclock, contents
else:
return None
def populate(self, Result):
"""
Populate the object based on the return from get.
If None returned, then object is not found
If a tuple of vclock, contents then one or more
whole revisions of the key were found
If a list of vtags is returned there are multiple
sibling that need to be retrieved with get.
"""
self.clear()
if Result is None:
return self
elif type(Result) == types.ListType:
self.set_siblings(Result)
elif type(Result) == types.TupleType:
(vclock, contents) = Result
self._vclock = vclock
if len(contents) > 0:
(metadata, data) = contents.pop(0)
self._exists = True
self.set_metadata(metadata)
self.set_encoded_data(data)
# Create objects for all siblings
siblings = [self]
for (metadata, data) in contents:
sibling = copy.copy(self)
sibling.set_metadata(metadata)
sibling.set_encoded_data(data)
siblings.append(sibling)
for sibling in siblings:
sibling.set_siblings(siblings)
else:
raise RiakError("do not know how to handle type " +
str(type(Result)))
def validate_timeseries_put_resp(self, resp_code, resp):
if resp is None and resp_code == MSG_CODE_TS_TTB_MSG:
return True
if resp is not None:
return True
else:
raise RiakError("missing response object")
def encode_timeseries_put(self, tsobj):
'''
Returns an Erlang-TTB encoded tuple with the appropriate data and
metadata from a TsObject.
:param tsobj: a TsObject
:type tsobj: TsObject
:rtype: term-to-binary encoded object
'''
if tsobj.columns:
raise NotImplementedError('columns are not used')
if tsobj.rows and isinstance(tsobj.rows, list):
req_rows = []
for row in tsobj.rows:
req_r = []
for cell in row:
req_r.append(self.encode_to_ts_cell(cell))
req_rows.append(tuple(req_r))
req = tsputreq_a, tsobj.table.name, [], req_rows
mc = MSG_CODE_TS_TTB_MSG
rc = MSG_CODE_TS_TTB_MSG
return Msg(mc, encode(req), rc)
else:
raise RiakError("TsObject requires a list of rows")
def stream_index(self, bucket, index, startkey, endkey=None,
return_terms=None, max_results=None, continuation=None,
timeout=None, term_regex=None):
"""
Streams a secondary index query.
"""
if not self.stream_indexes():
raise NotImplementedError("Secondary index streaming is not "
"supported on %s" %
self.server_version.vstring)
if term_regex and not self.index_term_regex():
raise NotImplementedError("Secondary index term_regex is not "
"supported on %s" %
self.server_version.vstring)
if timeout == 'infinity':
timeout = 0
params = {'return_terms': return_terms, 'stream': True,
'max_results': max_results, 'continuation': continuation,
'timeout': timeout, 'term_regex': term_regex}
bucket_type = self._get_bucket_type(bucket.bucket_type)
url = self.index_path(bucket.name, index, startkey, endkey,
bucket_type=bucket_type, **params)
status, headers, response = self._request('GET', url, stream=True)
if status == 200:
return RiakHttpIndexStream(response, index, return_terms)
else:
raise RiakError('Error streaming secondary index.')
def get(self, robj, r=None, pr=None, vtag=None):
"""
Serialize get request and deserialize response
"""
if vtag is not None:
raise RiakError("PB transport does not support vtags")
bucket = robj.bucket
req = riak_pb.RpbGetReq()
if r:
req.r = self.translate_rw_val(r)
if self.quorum_controls() and pr:
req.pr = self.translate_rw_val(pr)
if self.tombstone_vclocks():
req.deletedvclock = 1
req.bucket = bucket.name
req.key = robj.key
msg_code, resp = self._request(MSG_CODE_GET_REQ, req)
if msg_code == MSG_CODE_GET_RESP:
return self._decoded_contents(resp, robj)
else:
return None
def create_search_index(self, index, schema=None, n_val=None):
"""
Create a Solr search index for Yokozuna.
:param index: a name of a yz index
:type index: string
:param schema: XML of Solr schema
:type schema: string
:param n_val: N value of the write
:type n_val: int
:rtype boolean
"""
if not self.yz_wm_index:
raise NotImplementedError("Search 2.0 administration is not "
"supported for this version")
url = self.search_index_path(index)
headers = {'Content-Type': 'application/json'}
content_dict = dict()
if schema:
content_dict['schema'] = schema
if n_val:
content_dict['n_val'] = n_val
content = json.dumps(content_dict)
# Run the request...
status, _, _ = self._request('PUT', url, headers, content)
if status != 204:
raise RiakError('Error setting Search 2.0 index.')
return True
def set_client_id(self, client_id):
"""
Set the client id. This overrides the default, random client
id, which is automatically generated when none is specified in
when creating the transport object.
"""
raise RiakError("not implemented")
def put(self, robj, w=None, dw=None, return_body=True):
"""
Serialize put request and deserialize response - if 'content'
is true, retrieve the updated metadata/content
@return (vclock=None, [(metadata, value)]=None)
"""
raise RiakError("not implemented")
def test_bad_resource_inner_exception(self):
ex_msg = 'exception-message!'
ex = RiakError(ex_msg)
with self.assertRaises(BadResource) as cm:
raise BadResource(ex)
br_ex = cm.exception
self.assertEqual(br_ex.args[0], ex)
def remove_index(self, field=None, value=None):
"""
remove_index(field=None, value=None)
Remove the specified field/value pair as an index on this
object.
:param field: The index field.
:type field: string
:param value: The index value.
:type value: string or integer
:rtype: :class:`RiakObject <riak.riak_object.RiakObject>`
"""
if not field and not value:
self.indexes.clear()
elif field and not value:
for index in [x for x in self.indexes if x[0] == field]:
self.indexes.remove(index)
elif field and value:
self.indexes.remove((field, value))
else:
raise RiakError("Cannot pass value without a field"
" name while removing index")
return self._robject
def _encode_timeseries_put(self, tsobj, req):
"""
Fills an TsPutReq message with the appropriate data and
metadata from a TsObject.
:param tsobj: a TsObject
:type tsobj: TsObject
:param req: the protobuf message to fill
:type req: riak.pb.riak_ts_pb2.TsPutReq
"""
req.table = str_to_bytes(tsobj.table.name)
if tsobj.columns:
raise NotImplementedError("columns are not implemented yet")
if tsobj.rows and isinstance(tsobj.rows, list):
for row in tsobj.rows:
tsr = req.rows.add() # NB: type TsRow
if not isinstance(row, list):
raise ValueError("TsObject row must be a list of values")
for cell in row:
tsc = tsr.cells.add() # NB: type TsCell
self._encode_to_ts_cell(cell, tsc)
else:
raise RiakError("TsObject requires a list of rows")
def send_msg(self, msg_code, msg):
pkt = self.encode_msg(msg_code, msg)
sent_len = self._sock.send(pkt)
if sent_len != len(pkt):
raise RiakError(
"PB socket returned short write {0} - expected {1}".format(
sent_len, len(pkt)))
def get(self, robj, r=None, vtag=None):
"""
Serialize get request and deserialize response
"""
if vtag is not None:
raise RiakError("PB transport does not support vtags")
bucket = robj.get_bucket()
req = riakclient_pb2.RpbGetReq()
req.r = self.translate_rw_val(r)
req.bucket = bucket.get_name()
req.key = robj.get_key()
self.maybe_connect()
self.send_msg(MSG_CODE_GET_REQ, req)
msg_code, resp = self.recv_msg()
if msg_code == MSG_CODE_GET_RESP:
contents = []
for c in resp.content:
contents.append(self.decode_content(c))
return (resp.vclock, contents)
else:
return 0
return 0
def put(self, robj, w=None, dw=None, return_body=True):
"""
Serialize get request and deserialize response
"""
bucket = robj.get_bucket()
req = riakclient_pb2.RpbPutReq()
req.w = self.translate_rw_val(w)
req.dw = self.translate_rw_val(dw)
if return_body == True:
req.return_body = 1
req.bucket = bucket.get_name()
req.key = robj.get_key()
vclock = robj.vclock()
if vclock is not None:
req.vclock = vclock
self.pbify_content(robj.get_metadata(), robj.get_encoded_data(),
req.content)
self.maybe_connect()
self.send_msg(MSG_CODE_PUT_REQ, req)
msg_code, resp = self.recv_msg()
if msg_code != MSG_CODE_PUT_RESP:
raise RiakError("unexpected protocol buffer message code: ",
msg_code)
if resp is not None:
contents = []
for c in resp.content:
contents.append(self.decode_content(c))
return (resp.vclock, contents)
def decode_timeseries(self, resp_ttb, tsobj,
convert_timestamp=False):
"""
Fills an TsObject with the appropriate data and
metadata from a TTB-encoded TsGetResp / TsQueryResp.
:param resp_ttb: the decoded TTB data
:type resp_ttb: TTB-encoded tsqueryrsp or tsgetresp
:param tsobj: a TsObject
:type tsobj: TsObject
:param convert_timestamp: Convert timestamps to datetime objects
:type tsobj: boolean
"""
if resp_ttb is None:
return tsobj
self.maybe_err_ttb(resp_ttb)
# NB: some queries return a BARE 'tsqueryresp' atom
# catch that here:
if resp_ttb == tsqueryresp_a:
return tsobj
# The response atom is the first element in the response tuple
resp_a = resp_ttb[0]
if resp_a == tsputresp_a:
return
elif resp_a == tsgetresp_a or resp_a == tsqueryresp_a:
resp_data = resp_ttb[1]
if len(resp_data) == 0:
return
elif len(resp_data) == 3:
resp_colnames = resp_data[0]
resp_coltypes = resp_data[1]
tsobj.columns = self.decode_timeseries_cols(
resp_colnames, resp_coltypes)
resp_rows = resp_data[2]
tsobj.rows = []
for resp_row in resp_rows:
tsobj.rows.append(
self.decode_timeseries_row(resp_row, resp_coltypes,
convert_timestamp))
else:
raise RiakError(
"Expected 3-tuple in response, got: {}".format(resp_data))
else:
raise RiakError("Unknown TTB response type: {}".format(resp_a))
def _parse_body(self, robj, response, expected_statuses):
"""
Parse the body of an object response and populate the object.
"""
# If no response given, then return.
if response is None:
return None
status, headers, data = response
# Check if the server is down(status==0)
if not status:
m = 'Could not contact Riak Server: http://{0}:{1}!'.format(
self._node.host, self._node.http_port)
raise RiakError(m)
# Make sure expected code came back
self.check_http_code(status, expected_statuses)
if 'x-riak-vclock' in headers:
robj.vclock = VClock(headers['x-riak-vclock'], 'base64')
# If 404(Not Found), then clear the object.
if status == 404:
robj.siblings = []
return None
# If 201 Created, we need to extract the location and set the
# key on the object.
elif status == 201:
robj.key = headers['location'].strip().split('/')[-1]
# If 300(Siblings), apply the siblings to the object
elif status == 300:
ctype, params = parse_header(headers['content-type'])
if ctype == 'multipart/mixed':
if six.PY3:
data = bytes_to_str(data)
boundary = re.compile('\r?\n--%s(?:--)?\r?\n' %
re.escape(params['boundary']))
parts = [message_from_string(p)
for p in re.split(boundary, data)[1:-1]]
robj.siblings = [self._parse_sibling(RiakContent(robj),
part.items(),
part.get_payload())
for part in parts]
# Invoke sibling-resolution logic
if robj.resolver is not None:
robj.resolver(robj)
return robj
else:
raise Exception('unexpected sibling response format: {0}'.
format(ctype))
robj.siblings = [self._parse_sibling(RiakContent(robj),
headers.items(),
data)]
return robj
def set_bucket_props(self, bucket, props):
"""
Serialize set bucket property request and deserialize response
bucket = bucket object
props = dictionary of properties
@return boolean
"""
raise RiakError("not implemented")
def solr_select_path(self, index, query, **options):
if not self.riak_solr_searcher_wm and not self.yz_wm_search:
raise RiakError("Search is unsupported by this Riak node")
qs = {'q': query, 'wt': 'json', 'fl': '*,score'}
qs.update(options)
if index:
index = quote_plus(index)
return mkpath("/solr", index, "select", **qs)
def index_path(self, bucket, index, start, finish=None, **options):
if not self.riak_kv_wm_buckets:
raise RiakError("Indexes are unsupported by this Riak node")
if finish:
finish = quote_plus(str(finish))
return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket),
"index", quote_plus(index), quote_plus(str(start)),
finish, **options)
def datatypes_path(self, bucket_type, bucket, key=None, **options):
if not self.bucket_types():
raise RiakError("Datatypes are unsupported by this Riak node")
if key:
key = quote_plus(key)
return mkpath("/types", quote_plus(bucket_type), "buckets",
quote_plus(bucket), "datatypes", key, **options)
def put_new(self,
robj,
w=None,
dw=None,
pw=None,
return_body=True,
if_none_match=False):
"""Put a new object into the Riak store, returning its (new) key.
If return_meta is False, then the vlock and metadata return values
will be None.
@return robj
"""
# Note that this won't work on 0.14 nodes.
bucket = robj.bucket
req = riak_pb.RpbPutReq()
if w:
req.w = self.translate_rw_val(w)
if dw:
req.dw = self.translate_rw_val(dw)
if self.quorum_controls() and pw:
req.pw = self.translate_rw_val(pw)
if return_body:
req.return_body = 1
if if_none_match:
req.if_none_match = 1
req.bucket = bucket.name
self.encode_content(robj, req.content)
msg_code, resp = self._request(MSG_CODE_PUT_REQ, req,
MSG_CODE_PUT_RESP)
if not resp:
raise RiakError("missing response object")
if len(resp.content) != 1:
raise RiakError("siblings were returned from object creation")
robj.key = resp.key
robj.vclock = resp.vclock
content = self.decode_content(resp.content[0], robj)
return content
def recv_pkt(self):
nmsglen = self._sock.recv(4)
if (len(nmsglen) != 4):
raise RiakError(
"Socket returned short packet length {0} - expected 4".format(
nmsglen))
msglen, = struct.unpack('!i', nmsglen)
self._inbuf_len = msglen
self._inbuf = ''
while len(self._inbuf) < msglen:
want_len = min(8192, msglen - len(self._inbuf))
recv_buf = self._sock.recv(want_len)
if not recv_buf: break
self._inbuf += recv_buf
if len(self._inbuf) != self._inbuf_len:
raise RiakError(
"Socket returned short packet {0} - expected {1}".format(
len(self._inbuf), self._inbuf_len))
def ts_delete(self, table, key):
msg_code = riak.pb.messages.MSG_CODE_TS_DEL_REQ
codec = self._get_codec(msg_code)
msg = codec.encode_timeseries_keyreq(table, key, is_delete=True)
resp_code, resp = self._request(msg, codec)
if resp is not None:
return True
else:
raise RiakError("missing response object")
def put_new(self, robj, w=None, dw=None, return_meta=True):
"""Put a new object into the Riak store, returning its (new) key.
If return_meta is False, then the vlock and metadata return values
will be None.
@return (key, vclock, metadata)
"""
raise RiakError("not implemented")
