def writeRiak(action, bucketname, key, data):
global riak
logger = logging.getLogger("pullfromQueue")
bucket = riak.bucket(bucketname)
if action == "write":
obj = RiakObject(riak, bucket, key)
obj.content_type = "application/json"
obj.data = data
#index = None
#if index is not None:
# for indexKey in index:
# try:
# obj.add_index(indexKey,index[indexKey])
# except Exception as e:
# logger.error("Error updating index: "+e.message)
startTime = time.time()
storedObj = obj.store()
duration = round((time.time() - startTime),3)
if storedObj is not None:
results = storedObj.key
else:
results = "Not stored!"
logger.info(" Write "+bucketname+"/"+key+" Sz: "+str(len(json.dumps(data)))+" Dur: "+str(duration)+" Results: "+results)
elif action == "delete":
got = bucket.get(key)
startTime = time.time()
got.delete()
duration = round((time.time() - startTime),3)
logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
elif action == "test":
print(key)
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
# create key value pairs to stock in riak
key = str(str(pair[0]) + "_" + str(pair[1]))
val = {
"StockA": pair[0],
"StockB": pair[1],
"Date": pair[2],
"CloseA": pair[3],
"CloseB": pair[4],
"ZScore": pair[5],
"Beta": pair[6],
"SignalMean": pair[7],
"SignalSD": pair[8],
}
myDate = pair[2].split("-")
obj = RiakObject(rc, bucket, key)
# add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = "text/json"
obj.data = val
obj.data = json.dumps(val)
# store
obj.store()
# return a list of written pairs
return pair
def writeRiak(riak, action, bucketname, key, data, mimeType, logger, index=None):
bucket = riak.bucket(bucketname)
if action == "write":
obj = RiakObject(riak, bucket, key)
obj.content_type = mimeType
if mimeType == "application/octet-stream":
obj.encoded_data = data
else:
obj.data = data
if index is not None:
for indexKey in index:
try:
obj.add_index(indexKey,index[indexKey])
except Exception as e:
logger.error("Error updating index: "+e.message)
startTime = time.time()
storedObj = obj.store()
duration = round((time.time() - startTime),3)
if storedObj is not None:
results = storedObj.key
else:
results = "Not stored!"
logger.info(" Write "+(bucketname[-3:])+"/"+key+" Sz: "+str(len(data))+" Dur: "+str(duration)+" Results: "+results)
elif action == "delete":
got = bucket.get(key)
startTime = time.time()
got.delete()
duration = round((time.time() - startTime),3)
logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
#create key value pairs to stock in riak
key = str(str(pair[0]) + '_' + str(pair[1]))
val = {'StockA': pair[0], \
'StockB': pair[1], \
'Date': pair[2],\
'CloseA': pair[3], \
'CloseB': pair[4], \
'ZScore': pair[5],\
'Beta': pair[6],\
'SignalMean': pair[7],\
'SignalSD': pair[8]}
myDate = pair[2].split('-')
obj = RiakObject(rc, bucket, key)
#add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = 'text/json'
obj.data = val
obj.data = json.dumps(val)
#store
obj.store()
#return a list of written pairs
return pair
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket('stocks')
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def new(self,
key=None,
data=None,
content_type='application/json',
encoded_data=None):
"""A shortcut for manually instantiating a new
:class:`~riak.riak_object.RiakObject` or a new
:class:`~riak.datatypes.Datatype`, based on the presence and value
of the :attr:`datatype <BucketType.datatype>` bucket property. When
the bucket contains a :class:`~riak.datatypes.Datatype`, all
arguments are ignored except ``key``, otherwise they are used to
initialize the :class:`~riak.riak_object.RiakObject`.
:param key: Name of the key. Leaving this to be None (default)
will make Riak generate the key on store.
:type key: str
:param data: The data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.data <riak.riak_object.RiakObject.data>`.
:type data: object
:param content_type: The media type of the data stored in the
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.content_type
<riak.riak_object.RiakObject.content_type>`.
:type content_type: str
:param encoded_data: The encoded data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.encoded_data
<riak.riak_object.RiakObject.encoded_data>`.
:type encoded_data: str
:rtype: :class:`~riak.riak_object.RiakObject` or
:class:`~riak.datatypes.Datatype`
"""
from riak import RiakObject
if self.bucket_type.datatype:
return TYPES[self.bucket_type.datatype](bucket=self, key=key)
if PY2:
try:
if isinstance(data, string_types):
data = data.encode('ascii')
except UnicodeError:
raise TypeError('Unicode data values are not supported.')
obj = RiakObject(self._client, self, key)
obj.content_type = content_type
if data is not None:
obj.data = data
if encoded_data is not None:
obj.encoded_data = encoded_data
return obj
def new(self, key=None, data=None, content_type="application/json", encoded_data=None):
"""A shortcut for manually instantiating a new
:class:`~riak.riak_object.RiakObject` or a new
:class:`~riak.datatypes.Datatype`, based on the presence and value
of the :attr:`datatype <BucketType.datatype>` bucket property. When
the bucket contains a :class:`~riak.datatypes.Datatype`, all
arguments are ignored except ``key``, otherwise they are used to
initialize the :class:`~riak.riak_object.RiakObject`.
:param key: Name of the key. Leaving this to be None (default)
will make Riak generate the key on store.
:type key: str
:param data: The data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.data <riak.riak_object.RiakObject.data>`.
:type data: object
:param content_type: The media type of the data stored in the
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.content_type
<riak.riak_object.RiakObject.content_type>`.
:type content_type: str
:param encoded_data: The encoded data to store in a
:class:`~riak.riak_object.RiakObject`, see
:attr:`RiakObject.encoded_data
<riak.riak_object.RiakObject.encoded_data>`.
:type encoded_data: str
:rtype: :class:`~riak.riak_object.RiakObject` or
:class:`~riak.datatypes.Datatype`
"""
from riak import RiakObject
if self.bucket_type.datatype:
return TYPES[self.bucket_type.datatype](bucket=self, key=key)
if PY2:
try:
if isinstance(data, string_types):
data = data.encode("ascii")
except UnicodeError:
raise TypeError("Unicode data values are not supported.")
obj = RiakObject(self._client, self, key)
obj.content_type = content_type
if data is not None:
obj.data = data
if encoded_data is not None:
obj.encoded_data = encoded_data
return obj
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
# get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket("stocks")
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
# didnt get any data
if len(gtemp) == 0:
return 0
# got data
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
#get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket('stocks')
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
#didnt get any data
if len(gtemp) == 0:
return 0
#got data
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket("stocks")
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
