示例#1
0
def writeRiak(action, bucketname, key, data):
    global riak

    logger = logging.getLogger("pullfromQueue")
    bucket = riak.bucket(bucketname)
    if action == "write":
        obj = RiakObject(riak, bucket, key)
        obj.content_type = "application/json"
        obj.data = data
        #index = None
        #if index is not None:
        #        for indexKey in index:
        #            try:
        #                obj.add_index(indexKey,index[indexKey])
        #            except Exception as e:
         #               logger.error("Error updating index: "+e.message)
        startTime = time.time()
        storedObj = obj.store()
        duration = round((time.time() - startTime),3)
        if storedObj is not None:
            results = storedObj.key
        else:
            results = "Not stored!"
        logger.info(" Write "+bucketname+"/"+key+" Sz: "+str(len(json.dumps(data)))+" Dur: "+str(duration)+" Results: "+results)
    elif action == "delete":
        got = bucket.get(key)
        startTime = time.time()
        got.delete()
        duration = round((time.time() - startTime),3)
        logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
    elif action == "test":
        print(key)
示例#2
0
    def new(self,
            key=None,
            data=None,
            content_type='application/json',
            encoded_data=None):
        """A shortcut for manually instantiating a new
        :class:`~riak.riak_object.RiakObject` or a new
        :class:`~riak.datatypes.Datatype`, based on the presence and value
        of the :attr:`datatype <BucketType.datatype>` bucket property. When
        the bucket contains a :class:`~riak.datatypes.Datatype`, all
        arguments are ignored except ``key``, otherwise they are used to
        initialize the :class:`~riak.riak_object.RiakObject`.

        :param key: Name of the key. Leaving this to be None (default)
                    will make Riak generate the key on store.
        :type key: str
        :param data: The data to store in a
           :class:`~riak.riak_object.RiakObject`, see
           :attr:`RiakObject.data <riak.riak_object.RiakObject.data>`.
        :type data: object
        :param content_type: The media type of the data stored in the
           :class:`~riak.riak_object.RiakObject`, see
           :attr:`RiakObject.content_type
           <riak.riak_object.RiakObject.content_type>`.
        :type content_type: str
        :param encoded_data: The encoded data to store in a
           :class:`~riak.riak_object.RiakObject`, see
           :attr:`RiakObject.encoded_data
           <riak.riak_object.RiakObject.encoded_data>`.
        :type encoded_data: str
        :rtype: :class:`~riak.riak_object.RiakObject` or
                :class:`~riak.datatypes.Datatype`

        """
        from riak import RiakObject
        if self.bucket_type.datatype:
            return TYPES[self.bucket_type.datatype](bucket=self, key=key)

        if PY2:
            try:
                if isinstance(data, string_types):
                    data = data.encode('ascii')
            except UnicodeError:
                raise TypeError('Unicode data values are not supported.')

        obj = RiakObject(self._client, self, key)
        obj.content_type = content_type
        if data is not None:
            obj.data = data
        if encoded_data is not None:
            obj.encoded_data = encoded_data
        return obj
示例#3
0
    def get(self,
            key,
            r=None,
            pr=None,
            timeout=None,
            include_context=None,
            basic_quorum=None,
            notfound_ok=None):
        """
        Retrieve a :class:`~riak.riak_object.RiakObject` or
        :class:`~riak.datatypes.Datatype`, based on the presence and value
        of the :attr:`datatype <BucketType.datatype>` bucket property.

        :param key: Name of the key.
        :type key: string
        :param r: R-Value of the request (defaults to bucket's R)
        :type r: integer
        :param pr: PR-Value of the request (defaults to bucket's PR)
        :type pr: integer
        :param timeout: a timeout value in milliseconds
        :type timeout: int
        :param include_context: if the bucket contains datatypes, include
           the opaque context in the result
        :type include_context: bool
        :param basic_quorum: whether to use the "basic quorum" policy
           for not-founds
        :type basic_quorum: bool
        :param notfound_ok: whether to treat not-found responses as successful
        :type notfound_ok: bool
        :rtype: :class:`RiakObject <riak.riak_object.RiakObject>` or
           :class:`~riak.datatypes.Datatype`

        """
        from riak import RiakObject
        if self.bucket_type.datatype:
            return self._client.fetch_datatype(self,
                                               key,
                                               r=r,
                                               pr=pr,
                                               timeout=timeout,
                                               include_context=include_context,
                                               basic_quorum=basic_quorum,
                                               notfound_ok=notfound_ok)
        else:
            obj = RiakObject(self._client, self, key)
            return obj.reload(r=r,
                              pr=pr,
                              timeout=timeout,
                              basic_quorum=basic_quorum,
                              notfound_ok=notfound_ok)
示例#4
0
    def new(self, key=None, data=None, content_type="application/json", encoded_data=None):
        """A shortcut for manually instantiating a new
        :class:`~riak.riak_object.RiakObject` or a new
        :class:`~riak.datatypes.Datatype`, based on the presence and value
        of the :attr:`datatype <BucketType.datatype>` bucket property. When
        the bucket contains a :class:`~riak.datatypes.Datatype`, all
        arguments are ignored except ``key``, otherwise they are used to
        initialize the :class:`~riak.riak_object.RiakObject`.

        :param key: Name of the key. Leaving this to be None (default)
                    will make Riak generate the key on store.
        :type key: str
        :param data: The data to store in a
           :class:`~riak.riak_object.RiakObject`, see
           :attr:`RiakObject.data <riak.riak_object.RiakObject.data>`.
        :type data: object
        :param content_type: The media type of the data stored in the
           :class:`~riak.riak_object.RiakObject`, see
           :attr:`RiakObject.content_type
           <riak.riak_object.RiakObject.content_type>`.
        :type content_type: str
        :param encoded_data: The encoded data to store in a
           :class:`~riak.riak_object.RiakObject`, see
           :attr:`RiakObject.encoded_data
           <riak.riak_object.RiakObject.encoded_data>`.
        :type encoded_data: str
        :rtype: :class:`~riak.riak_object.RiakObject` or
                :class:`~riak.datatypes.Datatype`

        """
        from riak import RiakObject

        if self.bucket_type.datatype:
            return TYPES[self.bucket_type.datatype](bucket=self, key=key)

        if PY2:
            try:
                if isinstance(data, string_types):
                    data = data.encode("ascii")
            except UnicodeError:
                raise TypeError("Unicode data values are not supported.")

        obj = RiakObject(self._client, self, key)
        obj.content_type = content_type
        if data is not None:
            obj.data = data
        if encoded_data is not None:
            obj.encoded_data = encoded_data
        return obj
    def test_delete_datatype(self):
        ctype = self.client.bucket_type('counters')
        cbucket = ctype.bucket(self.bucket_name)
        counter = cbucket.new(self.key_name)
        counter.increment(5)
        counter.store()

        stype = self.client.bucket_type('sets')
        sbucket = stype.bucket(self.bucket_name)
        set_ = sbucket.new(self.key_name)
        set_.add("Brett")
        set_.store()

        mtype = self.client.bucket_type('maps')
        mbucket = mtype.bucket(self.bucket_name)
        map_ = mbucket.new(self.key_name)
        map_.sets['people'].add('Sean')
        map_.store()

        for t in [counter, set_, map_]:
            t.delete()
            obj = RiakObject(self.client, t.bucket, t.key)
            self.client.get(obj)
            self.assertFalse(obj.exists,
                             "{0} exists after deletion".format(t.type_name))
示例#6
0
def writeRiak(riak, action, bucketname, key, data, mimeType, logger, index=None):
	bucket = riak.bucket(bucketname)
	if action == "write":
		obj = RiakObject(riak, bucket, key)
		obj.content_type = mimeType
		if mimeType == "application/octet-stream":
			obj.encoded_data = data
		else:
			obj.data = data
		if index is not None:
				for indexKey in index:
					try:
						obj.add_index(indexKey,index[indexKey])
					except Exception as e:
						logger.error("Error updating index: "+e.message)
		startTime = time.time()
		storedObj = obj.store()
		duration = round((time.time() - startTime),3)
		if storedObj is not None:
			results = storedObj.key
		else:
			results = "Not stored!"
		logger.info(" Write "+(bucketname[-3:])+"/"+key+" Sz: "+str(len(data))+" Dur: "+str(duration)+" Results: "+results)
	elif action == "delete":
		got = bucket.get(key)
		startTime = time.time()
		got.delete()
		duration = round((time.time() - startTime),3)
		logger.info(" Block delete "+(bucketname[-3:])+"/"+key+" Duration: "+str(duration))
示例#7
0
    def get(self, key, r=None, pr=None, timeout=None, include_context=None,
            basic_quorum=None, notfound_ok=None, head_only=False):
        """
        Retrieve a :class:`~riak.riak_object.RiakObject` or
        :class:`~riak.datatypes.Datatype`, based on the presence and value
        of the :attr:`datatype <BucketType.datatype>` bucket property.

        :param key: Name of the key.
        :type key: string
        :param r: R-Value of the request (defaults to bucket's R)
        :type r: integer
        :param pr: PR-Value of the request (defaults to bucket's PR)
        :type pr: integer
        :param timeout: a timeout value in milliseconds
        :type timeout: int
        :param include_context: if the bucket contains datatypes, include
           the opaque context in the result
        :type include_context: bool
        :param basic_quorum: whether to use the "basic quorum" policy
           for not-founds
        :type basic_quorum: bool
        :param notfound_ok: whether to treat not-found responses as successful
        :type notfound_ok: bool
        :param head_only: whether to fetch without value, so only metadata
           (only available on PB transport)
        :type head_only: bool
        :rtype: :class:`RiakObject <riak.riak_object.RiakObject>` or
           :class:`~riak.datatypes.Datatype`

        """
        from riak import RiakObject
        if self.bucket_type.datatype:
            return self._client.fetch_datatype(self, key, r=r, pr=pr,
                                               timeout=timeout,
                                               include_context=include_context,
                                               basic_quorum=basic_quorum,
                                               notfound_ok=notfound_ok)
        else:
            obj = RiakObject(self._client, self, key)
            return obj.reload(r=r, pr=pr, timeout=timeout,
                              basic_quorum=basic_quorum,
                              notfound_ok=notfound_ok,
                              head_only=head_only)
示例#8
0
 def riak_object(self, modelcls, key, result=None):
     bucket = self.bucket_for_modelcls(modelcls)._riak_bucket
     riak_object = VumiTxRiakObject(RiakObject(self.client, bucket, key))
     if result:
         metadata = result['metadata']
         indexes = metadata['index']
         if hasattr(indexes, 'items'):
             # TODO: I think this is a Riak bug. In some cases
             #       (maybe when there are no indexes?) the index
             #       comes back as a list, in others (maybe when
             #       there are indexes?) it comes back as a dict.
             indexes = indexes.items()
         data = result['data']
         riak_object.set_content_type(metadata['content-type'])
         riak_object.set_indexes(indexes)
         riak_object.set_encoded_data(data)
     else:
         riak_object.set_content_type("application/json")
         riak_object.set_data({'$VERSION': modelcls.VERSION})
     return riak_object
示例#9
0
def writeHistory(ticker, data, riakIP):
    rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
    bucket = rc.bucket('stocks')
    gtemp = data
    if len(gtemp) == 0:
        return 0
    else:

        for j in range(0, len(gtemp.index)):

            #upload json to Riak Bucket
            date = gtemp.index[j].date()
            riakKey = str(ticker + '_' + str(date))
            riakVal = {'OPEN': gtemp.values[j,0],\
                        'HIGH': gtemp.values[j,1],\
                        'LOW': gtemp.values[j,2], \
                        'CLOSE': gtemp.values[j,3], \
                        'VOLUME': gtemp.values[j,4],\
                        'DATE': str(date),\
                        'TICKER': str(ticker)}

            obj = RiakObject(rc, bucket, riakKey)

            obj.add_index("ticker_bin", str(ticker))
            obj.add_index("year_int", int(date.year))
            obj.add_index("month_int", int(date.month))
            obj.add_index("day_int", int(date.day))

            obj.content_type = 'text/json'
            #obj.data = riakVal
            obj.data = json.dumps(riakVal)
            obj.store()

    return len(gtemp.index)
示例#10
0
def getDataByTicker(ticker, dataSource, start, end, riakIP):

    rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
    #get daily data for each ticker
    gtemp = pd.DataFrame()
    bucket = rc.bucket('stocks')
    try:
        gtemp = DataReader(ticker, dataSource, start, end)
        print ticker
    except:
        pass

        #didnt get any data
    if len(gtemp) == 0:
        return 0
    #got data
    else:

        for j in range(0, len(gtemp.index)):

            #upload json to Riak Bucket
            date = gtemp.index[j].date()
            riakKey = str(ticker + '_' + str(date))
            riakVal = {'OPEN': gtemp.values[j,0],\
                        'HIGH': gtemp.values[j,1],\
                        'LOW': gtemp.values[j,2], \
                        'CLOSE': gtemp.values[j,3], \
                        'VOLUME': gtemp.values[j,4],\
                        'DATE': str(date),\
                        'TICKER': str(ticker)}

            obj = RiakObject(rc, bucket, riakKey)

            obj.add_index("ticker_bin", str(ticker))
            obj.add_index("year_int", int(date.year))
            obj.add_index("month_int", int(date.month))
            obj.add_index("day_int", int(date.day))

            obj.content_type = 'text/json'
            #obj.data = riakVal
            obj.data = json.dumps(riakVal)
            obj.store()

    return len(gtemp.index)
示例#11
0
 def riak_object(self, modelcls, key, result=None):
     bucket = self.bucket_for_modelcls(modelcls)
     riak_object = RiakObject(self.client, bucket, key)
     if result:
         metadata = result['metadata']
         indexes = metadata['index']
         if hasattr(indexes, 'items'):
             # TODO: I think this is a Riak bug. In some cases
             #       (maybe when there are no indexes?) the index
             #       comes back as a list, in others (maybe when
             #       there are indexes?) it comes back as a dict.
             indexes = indexes.items()
         data = result['data']
         riak_object.set_content_type(metadata['content-type'])
         riak_object.set_indexes(indexes)
         riak_object.set_encoded_data(data)
     else:
         riak_object.set_data({'$VERSION': modelcls.VERSION})
         riak_object.set_content_type("application/json")
     return riak_object
#!/usr/bin/env python3

from riak import RiakClient, RiakObject, RiakBucket

riak_client = RiakClient(nodes=[{'host':'192.168.50.11','http_port':8098},
                                {'host':'192.168.50.12','http_port':8098},
                                {'host':'192.168.50.13','http_port':8098}])

bucket = riak_client.bucket('testbucket')

for object_id in range(100000):
    riak_object = RiakObject(riak_client, bucket, str(object_id))
    riak_object.data = {'id': object_id, 'desc': 'Description for ' + str(object_id)}
    riak_object.store()

示例#13
0
def create_test_data(client, n):
    """Helper function for creating test data."""
    for i in range(n):
        r = RiakObject(client, client.bucket("test.bucket.%d" % i))
        r.set_data({"foo": i, "bar": "baz"})
        r.store()
示例#14
0
 def delete(self, options):
     (bucket, key) = self.extract_bkey(options)
     result = RiakObject(self.client, self.client.bucket(bucket),
                         key).delete(rw=options.get('rw'))
     return True
示例#15
0
def writeSinglePair(pair, bucketName, riakIP):

    rc = RiakClient(protocol='pbc', host=riakIP, pb_port=8087)
    bucket = rc.bucket(bucketName)

    #create key value pairs to stock in riak
    key = str(str(pair[0]) + '_' + str(pair[1]))
    val = {'StockA': pair[0], \
                'StockB': pair[1], \
                'Date': pair[2],\
                'CloseA': pair[3], \
                'CloseB': pair[4], \
                'ZScore': pair[5],\
                'Beta': pair[6],\
                'SignalMean': pair[7],\
                'SignalSD': pair[8]}
    myDate = pair[2].split('-')
    obj = RiakObject(rc, bucket, key)

    #add 2i tags
    obj.add_index("stocka_bin", str(pair[0]))
    obj.add_index("stockb_bin", str(pair[3]))
    obj.add_index("year_int", int(myDate[0]))
    obj.add_index("month_int", int(myDate[1]))
    obj.add_index("day_int", int(myDate[2]))
    obj.content_type = 'text/json'
    obj.data = val
    obj.data = json.dumps(val)
    #store
    obj.store()

    #return a list of written pairs
    return pair
示例#16
0
def getDataByTicker(ticker, dataSource, start, end, riakIP):

    rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
    # get daily data for each ticker
    gtemp = pd.DataFrame()
    bucket = rc.bucket("stocks")
    try:
        gtemp = DataReader(ticker, dataSource, start, end)
        print ticker
    except:
        pass

        # didnt get any data
    if len(gtemp) == 0:
        return 0
    # got data
    else:

        for j in range(0, len(gtemp.index)):

            # upload json to Riak Bucket
            date = gtemp.index[j].date()
            riakKey = str(ticker + "_" + str(date))
            riakVal = {
                "OPEN": gtemp.values[j, 0],
                "HIGH": gtemp.values[j, 1],
                "LOW": gtemp.values[j, 2],
                "CLOSE": gtemp.values[j, 3],
                "VOLUME": gtemp.values[j, 4],
                "DATE": str(date),
                "TICKER": str(ticker),
            }

            obj = RiakObject(rc, bucket, riakKey)

            obj.add_index("ticker_bin", str(ticker))
            obj.add_index("year_int", int(date.year))
            obj.add_index("month_int", int(date.month))
            obj.add_index("day_int", int(date.day))

            obj.content_type = "text/json"
            # obj.data = riakVal
            obj.data = json.dumps(riakVal)
            obj.store()

    return len(gtemp.index)
示例#17
0
def writeSinglePair(pair, bucketName, riakIP):

    rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
    bucket = rc.bucket(bucketName)

    # create key value pairs to stock in riak
    key = str(str(pair[0]) + "_" + str(pair[1]))
    val = {
        "StockA": pair[0],
        "StockB": pair[1],
        "Date": pair[2],
        "CloseA": pair[3],
        "CloseB": pair[4],
        "ZScore": pair[5],
        "Beta": pair[6],
        "SignalMean": pair[7],
        "SignalSD": pair[8],
    }
    myDate = pair[2].split("-")
    obj = RiakObject(rc, bucket, key)

    # add 2i tags
    obj.add_index("stocka_bin", str(pair[0]))
    obj.add_index("stockb_bin", str(pair[3]))
    obj.add_index("year_int", int(myDate[0]))
    obj.add_index("month_int", int(myDate[1]))
    obj.add_index("day_int", int(myDate[2]))
    obj.content_type = "text/json"
    obj.data = val
    obj.data = json.dumps(val)
    # store
    obj.store()

    # return a list of written pairs
    return pair
示例#18
0
def writeHistory(ticker, data, riakIP):
    rc = RiakClient(protocol="pbc", host=riakIP, pb_port=8087)
    bucket = rc.bucket("stocks")
    gtemp = data
    if len(gtemp) == 0:
        return 0
    else:

        for j in range(0, len(gtemp.index)):

            # upload json to Riak Bucket
            date = gtemp.index[j].date()
            riakKey = str(ticker + "_" + str(date))
            riakVal = {
                "OPEN": gtemp.values[j, 0],
                "HIGH": gtemp.values[j, 1],
                "LOW": gtemp.values[j, 2],
                "CLOSE": gtemp.values[j, 3],
                "VOLUME": gtemp.values[j, 4],
                "DATE": str(date),
                "TICKER": str(ticker),
            }

            obj = RiakObject(rc, bucket, riakKey)

            obj.add_index("ticker_bin", str(ticker))
            obj.add_index("year_int", int(date.year))
            obj.add_index("month_int", int(date.month))
            obj.add_index("day_int", int(date.day))

            obj.content_type = "text/json"
            # obj.data = riakVal
            obj.data = json.dumps(riakVal)
            obj.store()

    return len(gtemp.index)