def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
# create key value pairs to stock in riak
key = str(str(pair[0]) + "_" + str(pair[1]))
val = {
"StockA": pair[0],
"StockB": pair[1],
"Date": pair[2],
"CloseA": pair[3],
"CloseB": pair[4],
"ZScore": pair[5],
"Beta": pair[6],
"SignalMean": pair[7],
"SignalSD": pair[8],
}
myDate = pair[2].split("-")
obj = RiakObject(rc, bucket, key)
# add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = "text/json"
obj.data = val
obj.data = json.dumps(val)
# store
obj.store()
# return a list of written pairs
return pair
def writeSinglePair(pair, bucketName, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket(bucketName)
#create key value pairs to stock in riak
key = str(str(pair[0]) + '_' + str(pair[1]))
val = {'StockA': pair[0], \
'StockB': pair[1], \
'Date': pair[2],\
'CloseA': pair[3], \
'CloseB': pair[4], \
'ZScore': pair[5],\
'Beta': pair[6],\
'SignalMean': pair[7],\
'SignalSD': pair[8]}
myDate = pair[2].split('-')
obj = RiakObject(rc, bucket, key)
#add 2i tags
obj.add_index("stocka_bin", str(pair[0]))
obj.add_index("stockb_bin", str(pair[3]))
obj.add_index("year_int", int(myDate[0]))
obj.add_index("month_int", int(myDate[1]))
obj.add_index("day_int", int(myDate[2]))
obj.content_type = 'text/json'
obj.data = val
obj.data = json.dumps(val)
#store
obj.store()
#return a list of written pairs
return pair
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
bucket = rc.bucket('stocks')
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def writeRiak(action, bucketname, key, data):
global riak
logger = logging.getLogger("pullfromQueue")
bucket = riak.bucket(bucketname)
if action == "write":
obj = RiakObject(riak, bucket, key)
obj.content_type = "application/json"
obj.data = data
#index = None
#if index is not None:
# for indexKey in index:
# try:
# obj.add_index(indexKey,index[indexKey])
# except Exception as e:
# logger.error("Error updating index: "+e.message)
startTime = time.time()
storedObj = obj.store()
duration = round((time.time() - startTime),3)
if storedObj is not None:
results = storedObj.key
else:
results = "Not stored!"
logger.info(" Write "+bucketname+"/"+key+" Sz: "+str(len(json.dumps(data)))+" Dur: "+str(duration)+" Results: "+results)
elif action == "delete":
got = bucket.get(key)
startTime = time.time()
got.delete()
duration = round((time.time() - startTime),3)
logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
elif action == "test":
print(key)
def writeRiak(riak, action, bucketname, key, data, mimeType, logger, index=None):
bucket = riak.bucket(bucketname)
if action == "write":
obj = RiakObject(riak, bucket, key)
obj.content_type = mimeType
if mimeType == "application/octet-stream":
obj.encoded_data = data
else:
obj.data = data
if index is not None:
for indexKey in index:
try:
obj.add_index(indexKey,index[indexKey])
except Exception as e:
logger.error("Error updating index: "+e.message)
startTime = time.time()
storedObj = obj.store()
duration = round((time.time() - startTime),3)
if storedObj is not None:
results = storedObj.key
else:
results = "Not stored!"
logger.info(" Write "+(bucketname[-3:])+"/"+key+" Sz: "+str(len(data))+" Dur: "+str(duration)+" Results: "+results)
elif action == "delete":
got = bucket.get(key)
startTime = time.time()
got.delete()
duration = round((time.time() - startTime),3)
logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
# get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket("stocks")
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
# didnt get any data
if len(gtemp) == 0:
return 0
# got data
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def getDataByTicker(ticker, dataSource, start, end, riakIP):
rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
#get daily data for each ticker
gtemp = pd.DataFrame()
bucket = rc.bucket('stocks')
try:
gtemp = DataReader(ticker, dataSource, start, end)
print ticker
except:
pass
#didnt get any data
if len(gtemp) == 0:
return 0
#got data
else:
for j in range(0, len(gtemp.index)):
#upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + '_' + str(date))
riakVal = {'OPEN': gtemp.values[j,0],\
'HIGH': gtemp.values[j,1],\
'LOW': gtemp.values[j,2], \
'CLOSE': gtemp.values[j,3], \
'VOLUME': gtemp.values[j,4],\
'DATE': str(date),\
'TICKER': str(ticker)}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = 'text/json'
#obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
def writeHistory(ticker, data, riakIP):
rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
bucket = rc.bucket("stocks")
gtemp = data
if len(gtemp) == 0:
return 0
else:
for j in range(0, len(gtemp.index)):
# upload json to Riak Bucket
date = gtemp.index[j].date()
riakKey = str(ticker + "_" + str(date))
riakVal = {
"OPEN": gtemp.values[j, 0],
"HIGH": gtemp.values[j, 1],
"LOW": gtemp.values[j, 2],
"CLOSE": gtemp.values[j, 3],
"VOLUME": gtemp.values[j, 4],
"DATE": str(date),
"TICKER": str(ticker),
}
obj = RiakObject(rc, bucket, riakKey)
obj.add_index("ticker_bin", str(ticker))
obj.add_index("year_int", int(date.year))
obj.add_index("month_int", int(date.month))
obj.add_index("day_int", int(date.day))
obj.content_type = "text/json"
# obj.data = riakVal
obj.data = json.dumps(riakVal)
obj.store()
return len(gtemp.index)
#!/usr/bin/env python3
from riak import RiakClient, RiakObject, RiakBucket
riak_client = RiakClient(nodes=[{'host':'192.168.50.11','http_port':8098},
{'host':'192.168.50.12','http_port':8098},
{'host':'192.168.50.13','http_port':8098}])
bucket = riak_client.bucket('testbucket')
for object_id in range(100000):
riak_object = RiakObject(riak_client, bucket, str(object_id))
riak_object.data = {'id': object_id, 'desc': 'Description for ' + str(object_id)}
riak_object.store()
def create_test_data(client, n):
"""Helper function for creating test data."""
for i in range(n):
r = RiakObject(client, client.bucket("test.bucket.%d" % i))
r.set_data({"foo": i, "bar": "baz"})
r.store()