def writeRiak(action, bucketname, key, data): global riak logger = logging.getLogger("pullfromQueue") bucket = riak.bucket(bucketname) if action == "write": obj = RiakObject(riak, bucket, key) obj.content_type = "application/json" obj.data = data #index = None #if index is not None: # for indexKey in index: # try: # obj.add_index(indexKey,index[indexKey]) # except Exception as e: # logger.error("Error updating index: "+e.message) startTime = time.time() storedObj = obj.store() duration = round((time.time() - startTime),3) if storedObj is not None: results = storedObj.key else: results = "Not stored!" logger.info(" Write "+bucketname+"/"+key+" Sz: "+str(len(json.dumps(data)))+" Dur: "+str(duration)+" Results: "+results) elif action == "delete": got = bucket.get(key) startTime = time.time() got.delete() duration = round((time.time() - startTime),3) logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration)) elif action == "test": print(key)
def writeSinglePair(pair, bucketName, riakIP): rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087) bucket = rc.bucket(bucketName) # create key value pairs to stock in riak key = str(str(pair[0]) + "_" + str(pair[1])) val = { "StockA": pair[0], "StockB": pair[1], "Date": pair[2], "CloseA": pair[3], "CloseB": pair[4], "ZScore": pair[5], "Beta": pair[6], "SignalMean": pair[7], "SignalSD": pair[8], } myDate = pair[2].split("-") obj = RiakObject(rc, bucket, key) # add 2i tags obj.add_index("stocka_bin", str(pair[0])) obj.add_index("stockb_bin", str(pair[3])) obj.add_index("year_int", int(myDate[0])) obj.add_index("month_int", int(myDate[1])) obj.add_index("day_int", int(myDate[2])) obj.content_type = "text/json" obj.data = val obj.data = json.dumps(val) # store obj.store() # return a list of written pairs return pair
def writeRiak(riak, action, bucketname, key, data, mimeType, logger, index=None): bucket = riak.bucket(bucketname) if action == "write": obj = RiakObject(riak, bucket, key) obj.content_type = mimeType if mimeType == "application/octet-stream": obj.encoded_data = data else: obj.data = data if index is not None: for indexKey in index: try: obj.add_index(indexKey,index[indexKey]) except Exception as e: logger.error("Error updating index: "+e.message) startTime = time.time() storedObj = obj.store() duration = round((time.time() - startTime),3) if storedObj is not None: results = storedObj.key else: results = "Not stored!" logger.info(" Write "+(bucketname[-3:])+"/"+key+" Sz: "+str(len(data))+" Dur: "+str(duration)+" Results: "+results) elif action == "delete": got = bucket.get(key) startTime = time.time() got.delete() duration = round((time.time() - startTime),3) logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
def writeSinglePair(pair, bucketName, riakIP): rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087) bucket = rc.bucket(bucketName) #create key value pairs to stock in riak key = str(str(pair[0]) + '_' + str(pair[1])) val = {'StockA': pair[0], \ 'StockB': pair[1], \ 'Date': pair[2],\ 'CloseA': pair[3], \ 'CloseB': pair[4], \ 'ZScore': pair[5],\ 'Beta': pair[6],\ 'SignalMean': pair[7],\ 'SignalSD': pair[8]} myDate = pair[2].split('-') obj = RiakObject(rc, bucket, key) #add 2i tags obj.add_index("stocka_bin", str(pair[0])) obj.add_index("stockb_bin", str(pair[3])) obj.add_index("year_int", int(myDate[0])) obj.add_index("month_int", int(myDate[1])) obj.add_index("day_int", int(myDate[2])) obj.content_type = 'text/json' obj.data = val obj.data = json.dumps(val) #store obj.store() #return a list of written pairs return pair
def writeHistory(ticker, data, riakIP): rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087) bucket = rc.bucket('stocks') gtemp = data if len(gtemp) == 0: return 0 else: for j in range(0, len(gtemp.index)): #upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + '_' + str(date)) riakVal = {'OPEN': gtemp.values[j,0],\ 'HIGH': gtemp.values[j,1],\ 'LOW': gtemp.values[j,2], \ 'CLOSE': gtemp.values[j,3], \ 'VOLUME': gtemp.values[j,4],\ 'DATE': str(date),\ 'TICKER': str(ticker)} obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = 'text/json' #obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
def new(self, key=None, data=None, content_type='application/json', encoded_data=None): """A shortcut for manually instantiating a new :class:`~riak.riak_object.RiakObject` or a new :class:`~riak.datatypes.Datatype`, based on the presence and value of the :attr:`datatype <BucketType.datatype>` bucket property. When the bucket contains a :class:`~riak.datatypes.Datatype`, all arguments are ignored except ``key``, otherwise they are used to initialize the :class:`~riak.riak_object.RiakObject`. :param key: Name of the key. Leaving this to be None (default) will make Riak generate the key on store. :type key: str :param data: The data to store in a :class:`~riak.riak_object.RiakObject`, see :attr:`RiakObject.data <riak.riak_object.RiakObject.data>`. :type data: object :param content_type: The media type of the data stored in the :class:`~riak.riak_object.RiakObject`, see :attr:`RiakObject.content_type <riak.riak_object.RiakObject.content_type>`. :type content_type: str :param encoded_data: The encoded data to store in a :class:`~riak.riak_object.RiakObject`, see :attr:`RiakObject.encoded_data <riak.riak_object.RiakObject.encoded_data>`. :type encoded_data: str :rtype: :class:`~riak.riak_object.RiakObject` or :class:`~riak.datatypes.Datatype` """ from riak import RiakObject if self.bucket_type.datatype: return TYPES[self.bucket_type.datatype](bucket=self, key=key) if PY2: try: if isinstance(data, string_types): data = data.encode('ascii') except UnicodeError: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) obj.content_type = content_type if data is not None: obj.data = data if encoded_data is not None: obj.encoded_data = encoded_data return obj
def new(self, key=None, data=None, content_type="application/json", encoded_data=None): """A shortcut for manually instantiating a new :class:`~riak.riak_object.RiakObject` or a new :class:`~riak.datatypes.Datatype`, based on the presence and value of the :attr:`datatype <BucketType.datatype>` bucket property. When the bucket contains a :class:`~riak.datatypes.Datatype`, all arguments are ignored except ``key``, otherwise they are used to initialize the :class:`~riak.riak_object.RiakObject`. :param key: Name of the key. Leaving this to be None (default) will make Riak generate the key on store. :type key: str :param data: The data to store in a :class:`~riak.riak_object.RiakObject`, see :attr:`RiakObject.data <riak.riak_object.RiakObject.data>`. :type data: object :param content_type: The media type of the data stored in the :class:`~riak.riak_object.RiakObject`, see :attr:`RiakObject.content_type <riak.riak_object.RiakObject.content_type>`. :type content_type: str :param encoded_data: The encoded data to store in a :class:`~riak.riak_object.RiakObject`, see :attr:`RiakObject.encoded_data <riak.riak_object.RiakObject.encoded_data>`. :type encoded_data: str :rtype: :class:`~riak.riak_object.RiakObject` or :class:`~riak.datatypes.Datatype` """ from riak import RiakObject if self.bucket_type.datatype: return TYPES[self.bucket_type.datatype](bucket=self, key=key) if PY2: try: if isinstance(data, string_types): data = data.encode("ascii") except UnicodeError: raise TypeError("Unicode data values are not supported.") obj = RiakObject(self._client, self, key) obj.content_type = content_type if data is not None: obj.data = data if encoded_data is not None: obj.encoded_data = encoded_data return obj
def getDataByTicker(ticker, dataSource, start, end, riakIP): rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087) # get daily data for each ticker gtemp = pd.DataFrame() bucket = rc.bucket("stocks") try: gtemp = DataReader(ticker, dataSource, start, end) print ticker except: pass # didnt get any data if len(gtemp) == 0: return 0 # got data else: for j in range(0, len(gtemp.index)): # upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + "_" + str(date)) riakVal = { "OPEN": gtemp.values[j, 0], "HIGH": gtemp.values[j, 1], "LOW": gtemp.values[j, 2], "CLOSE": gtemp.values[j, 3], "VOLUME": gtemp.values[j, 4], "DATE": str(date), "TICKER": str(ticker), } obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = "text/json" # obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
def getDataByTicker(ticker, dataSource, start, end, riakIP): rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087) #get daily data for each ticker gtemp = pd.DataFrame() bucket = rc.bucket('stocks') try: gtemp = DataReader(ticker, dataSource, start, end) print ticker except: pass #didnt get any data if len(gtemp) == 0: return 0 #got data else: for j in range(0, len(gtemp.index)): #upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + '_' + str(date)) riakVal = {'OPEN': gtemp.values[j,0],\ 'HIGH': gtemp.values[j,1],\ 'LOW': gtemp.values[j,2], \ 'CLOSE': gtemp.values[j,3], \ 'VOLUME': gtemp.values[j,4],\ 'DATE': str(date),\ 'TICKER': str(ticker)} obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = 'text/json' #obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)
def writeHistory(ticker, data, riakIP): rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087) bucket = rc.bucket("stocks") gtemp = data if len(gtemp) == 0: return 0 else: for j in range(0, len(gtemp.index)): # upload json to Riak Bucket date = gtemp.index[j].date() riakKey = str(ticker + "_" + str(date)) riakVal = { "OPEN": gtemp.values[j, 0], "HIGH": gtemp.values[j, 1], "LOW": gtemp.values[j, 2], "CLOSE": gtemp.values[j, 3], "VOLUME": gtemp.values[j, 4], "DATE": str(date), "TICKER": str(ticker), } obj = RiakObject(rc, bucket, riakKey) obj.add_index("ticker_bin", str(ticker)) obj.add_index("year_int", int(date.year)) obj.add_index("month_int", int(date.month)) obj.add_index("day_int", int(date.day)) obj.content_type = "text/json" # obj.data = riakVal obj.data = json.dumps(riakVal) obj.store() return len(gtemp.index)