def writeRiak(action, bucketname, key, data):
global riak
logger = logging.getLogger("pullfromQueue")
bucket = riak.bucket(bucketname)
if action == "write":
obj = RiakObject(riak, bucket, key)
obj.content_type = "application/json"
obj.data = data
#index = None
#if index is not None:
# for indexKey in index:
# try:
# obj.add_index(indexKey,index[indexKey])
# except Exception as e:
# logger.error("Error updating index: "+e.message)
startTime = time.time()
storedObj = obj.store()
duration = round((time.time() - startTime),3)
if storedObj is not None:
results = storedObj.key
else:
results = "Not stored!"
logger.info(" Write "+bucketname+"/"+key+" Sz: "+str(len(json.dumps(data)))+" Dur: "+str(duration)+" Results: "+results)
elif action == "delete":
got = bucket.get(key)
startTime = time.time()
got.delete()
duration = round((time.time() - startTime),3)
logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
elif action == "test":
print(key)
def new(self,
key=None,
data=None,
content_type='application/json',
encoded_data=None):
"""A shortcut for manually instantiating a new
:class:`~riak.riak_object.RiakObject` or a new
:class:`~riak.datatypes.Datatype`, based on the presence and value
of the :attr:`datatype <BucketType.datatype>` bucket property. When
the bucket contains a :class:`~riak.datatypes.Datatype`, all
arguments are ignored except ``key``, otherwise they are used to
initialize the :class:`~riak.riak_object.RiakObject`.
:param key: Name of the key. Leaving this to be None (default)
will make Riak generate the key on store.
:type key: str
:param data: The data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.data <riak.riak_object.RiakObject.data>`.
:type data: object
:param content_type: The media type of the data stored in the
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.content_type
<riak.riak_object.RiakObject.content_type>`.
:type content_type: str
:param encoded_data: The encoded data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.encoded_data
<riak.riak_object.RiakObject.encoded_data>`.
:type encoded_data: str
:rtype: :class:`~riak.riak_object.RiakObject` or
:class:`~riak.datatypes.Datatype`
"""
from riak import RiakObject
if self.bucket_type.datatype:
return TYPES[self.bucket_type.datatype](bucket=self, key=key)
if PY2:
try:
if isinstance(data, string_types):
data = data.encode('ascii')
except UnicodeError:
raise TypeError('Unicode data values are not supported.')
obj = RiakObject(self._client, self, key)
obj.content_type = content_type
if data is not None:
obj.data = data
if encoded_data is not None:
obj.encoded_data = encoded_data
return obj
def get(self,
key,
r=None,
pr=None,
timeout=None,
include_context=None,
basic_quorum=None,
notfound_ok=None):
"""
Retrieve a :class:`~riak.riak_object.RiakObject` or
:class:`~riak.datatypes.Datatype`, based on the presence and value
of the :attr:`datatype <BucketType.datatype>` bucket property.
:param key: Name of the key.
:type key: string
:param r: R-Value of the request (defaults to bucket's R)
:type r: integer
:param pr: PR-Value of the request (defaults to bucket's PR)
:type pr: integer
:param timeout: a timeout value in milliseconds
:type timeout: int
:param include_context: if the bucket contains datatypes, include
the opaque context in the result
:type include_context: bool
:param basic_quorum: whether to use the "basic quorum" policy
for not-founds
:type basic_quorum: bool
:param notfound_ok: whether to treat not-found responses as successful
:type notfound_ok: bool
:rtype: :class:`RiakObject <riak.riak_object.RiakObject>` or
:class:`~riak.datatypes.Datatype`
"""
from riak import RiakObject
if self.bucket_type.datatype:
return self._client.fetch_datatype(self,
key,
r=r,
pr=pr,
timeout=timeout,
include_context=include_context,
basic_quorum=basic_quorum,
notfound_ok=notfound_ok)
else:
obj = RiakObject(self._client, self, key)
return obj.reload(r=r,
pr=pr,
timeout=timeout,
basic_quorum=basic_quorum,
notfound_ok=notfound_ok)
def new(self, key=None, data=None, content_type="application/json", encoded_data=None):
"""A shortcut for manually instantiating a new
:class:`~riak.riak_object.RiakObject` or a new
:class:`~riak.datatypes.Datatype`, based on the presence and value
of the :attr:`datatype <BucketType.datatype>` bucket property. When
the bucket contains a :class:`~riak.datatypes.Datatype`, all
arguments are ignored except ``key``, otherwise they are used to
initialize the :class:`~riak.riak_object.RiakObject`.
:param key: Name of the key. Leaving this to be None (default)
will make Riak generate the key on store.
:type key: str
:param data: The data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.data <riak.riak_object.RiakObject.data>`.
:type data: object
:param content_type: The media type of the data stored in the
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.content_type
<riak.riak_object.RiakObject.content_type>`.
:type content_type: str
:param encoded_data: The encoded data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.encoded_data
<riak.riak_object.RiakObject.encoded_data>`.
:type encoded_data: str
:rtype: :class:`~riak.riak_object.RiakObject` or
:class:`~riak.datatypes.Datatype`
"""
from riak import RiakObject
if self.bucket_type.datatype:
return TYPES[self.bucket_type.datatype](bucket=self, key=key)
if PY2:
try:
if isinstance(data, string_types):
data = data.encode("ascii")
except UnicodeError:
raise TypeError("Unicode data values are not supported.")
obj = RiakObject(self._client, self, key)
obj.content_type = content_type
if data is not None:
obj.data = data
if encoded_data is not None:
obj.encoded_data = encoded_data
return obj
def test_delete_datatype(self):
ctype = self.client.bucket_type('counters')
cbucket = ctype.bucket(self.bucket_name)
counter = cbucket.new(self.key_name)
counter.increment(5)
counter.store()
stype = self.client.bucket_type('sets')
sbucket = stype.bucket(self.bucket_name)
set_ = sbucket.new(self.key_name)
set_.add("Brett")
set_.store()
mtype = self.client.bucket_type('maps')
mbucket = mtype.bucket(self.bucket_name)
map_ = mbucket.new(self.key_name)
map_.sets['people'].add('Sean')
map_.store()
for t in [counter, set_, map_]:
t.delete()
obj = RiakObject(self.client, t.bucket, t.key)
self.client.get(obj)
self.assertFalse(obj.exists,
"{0} exists after deletion".format(t.type_name))
def writeRiak(riak, action, bucketname, key, data, mimeType, logger, index=None):
bucket = riak.bucket(bucketname)
if action == "write":
obj = RiakObject(riak, bucket, key)
obj.content_type = mimeType
if mimeType == "application/octet-stream":
obj.encoded_data = data
else:
obj.data = data
if index is not None:
for indexKey in index:
try:
obj.add_index(indexKey,index[indexKey])
except Exception as e:
logger.error("Error updating index: "+e.message)
startTime = time.time()
storedObj = obj.store()
duration = round((time.time() - startTime),3)
if storedObj is not None:
results = storedObj.key
else:
results = "Not stored!"
logger.info(" Write "+(bucketname[-3:])+"/"+key+" Sz: "+str(len(data))+" Dur: "+str(duration)+" Results: "+results)
elif action == "delete":
got = bucket.get(key)
startTime = time.time()
got.delete()
duration = round((time.time() - startTime),3)
logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
def get(self, key, r=None, pr=None, timeout=None, include_context=None,
basic_quorum=None, notfound_ok=None, head_only=False):
"""
Retrieve a :class:`~riak.riak_object.RiakObject` or
:class:`~riak.datatypes.Datatype`, based on the presence and value
of the :attr:`datatype <BucketType.datatype>` bucket property.
:param key: Name of the key.
:type key: string
:param r: R-Value of the request (defaults to bucket's R)
:type r: integer
:param pr: PR-Value of the request (defaults to bucket's PR)
:type pr: integer
:param timeout: a timeout value in milliseconds
:type timeout: int
:param include_context: if the bucket contains datatypes, include
the opaque context in the result
:type include_context: bool
:param basic_quorum: whether to use the "basic quorum" policy
for not-founds
:type basic_quorum: bool
:param notfound_ok: whether to treat not-found responses as successful
:type notfound_ok: bool
:param head_only: whether to fetch without value, so only metadata
(only available on PB transport)
:type head_only: bool
:rtype: :class:`RiakObject <riak.riak_object.RiakObject>` or
:class:`~riak.datatypes.Datatype`
"""
from riak import RiakObject
if self.bucket_type.datatype:
return self._client.fetch_datatype(self, key, r=r, pr=pr,
timeout=timeout,
include_context=include_context,
basic_quorum=basic_quorum,
notfound_ok=notfound_ok)
else:
obj = RiakObject(self._client, self, key)
return obj.reload(r=r, pr=pr, timeout=timeout,
basic_quorum=basic_quorum,
notfound_ok=notfound_ok,
head_only=head_only)
def riak_object(self, modelcls, key, result=None):
bucket = self.bucket_for_modelcls(modelcls)._riak_bucket
riak_object = VumiTxRiakObject(RiakObject(self.client, bucket, key))
if result:
metadata = result['metadata']
indexes = metadata['index']
if hasattr(indexes, 'items'):
# TODO: I think this is a Riak bug. In some cases
# (maybe when there are no indexes?) the index
# comes back as a list, in others (maybe when
# there are indexes?) it comes back as a dict.
indexes = indexes.items()
data = result['data']
riak_object.set_content_type(metadata['content-type'])
riak_object.set_indexes(indexes)
riak_object.set_encoded_data(data)
else:
riak_object.set_content_type("application/json")
riak_object.set_data({'$VERSION': modelcls.VERSION})
return riak_object
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket('stocks')
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
#get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket('stocks')
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
#didnt get any data
if len(gtemp) == 0:
return 0
#got data
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def riak_object(self, modelcls, key, result=None):
bucket = self.bucket_for_modelcls(modelcls)
riak_object = RiakObject(self.client, bucket, key)
if result:
metadata = result['metadata']
indexes = metadata['index']
if hasattr(indexes, 'items'):
# TODO: I think this is a Riak bug. In some cases
# (maybe when there are no indexes?) the index
# comes back as a list, in others (maybe when
# there are indexes?) it comes back as a dict.
indexes = indexes.items()
data = result['data']
riak_object.set_content_type(metadata['content-type'])
riak_object.set_indexes(indexes)
riak_object.set_encoded_data(data)
else:
riak_object.set_data({'$VERSION': modelcls.VERSION})
riak_object.set_content_type("application/json")
return riak_object
#!/usr/bin/env python3
from riak import RiakClient, RiakObject, RiakBucket
riak_client = RiakClient(nodes=[{'host':'192.168.50.11','http_port':8098},
{'host':'192.168.50.12','http_port':8098},
{'host':'192.168.50.13','http_port':8098}])
bucket = riak_client.bucket('testbucket')
for object_id in range(100000):
riak_object = RiakObject(riak_client, bucket, str(object_id))
riak_object.data = {'id': object_id, 'desc': 'Description for ' + str(object_id)}
riak_object.store()
def create_test_data(client, n):
"""Helper function for creating test data."""
for i in range(n):
r = RiakObject(client, client.bucket("test.bucket.%d" % i))
r.set_data({"foo": i, "bar": "baz"})
r.store()
def delete(self, options):
(bucket, key) = self.extract_bkey(options)
result = RiakObject(self.client, self.client.bucket(bucket),
key).delete(rw=options.get('rw'))
return True
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
#create key value pairs to stock in riak
key = str(str(pair[0]) + '_' + str(pair[1]))
val = {'StockA': pair[0], \
'StockB': pair[1], \
'Date': pair[2],\
'CloseA': pair[3], \
'CloseB': pair[4], \
'ZScore': pair[5],\
'Beta': pair[6],\
'SignalMean': pair[7],\
'SignalSD': pair[8]}
myDate = pair[2].split('-')
obj = RiakObject(rc, bucket, key)
#add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = 'text/json'
obj.data = val
obj.data = json.dumps(val)
#store
obj.store()
#return a list of written pairs
return pair
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
# get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket("stocks")
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
# didnt get any data
if len(gtemp) == 0:
return 0
# got data
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
# create key value pairs to stock in riak
key = str(str(pair[0]) + "_" + str(pair[1]))
val = {
"StockA": pair[0],
"StockB": pair[1],
"Date": pair[2],
"CloseA": pair[3],
"CloseB": pair[4],
"ZScore": pair[5],
"Beta": pair[6],
"SignalMean": pair[7],
"SignalSD": pair[8],
}
myDate = pair[2].split("-")
obj = RiakObject(rc, bucket, key)
# add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = "text/json"
obj.data = val
obj.data = json.dumps(val)
# store
obj.store()
# return a list of written pairs
return pair
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket("stocks")
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)