Example #1
0
    def test_mergedsum(self):
        q = "nansum(axis=1) < paste < swindow(first, 20)"
        op = parse_opex(q).ast
        inst = op([{
            'uuid': str(uuid.uuid1()),
            'Properties/Timezone': 'America/Los_Angeles'
        }, {
            'uuid': str(uuid.uuid1()),
            'Properties/Timezone': 'America/Los_Angeles'
        }, {
            'uuid': str(uuid.uuid1()),
            'Properties/Timezone': 'America/Los_Angeles'
        }, {
            'uuid': str(uuid.uuid1()),
            'Properties/Timezone': 'America/Los_Angeles'
        }])
        self.assertFalse(inst.block_streaming)
        self.assertEquals(len(inst.outputs), 1)

        for i in [0, 1, 20, 21, 40, 41, 60, 61, 100, 101, 200, 300]:
            d = [operators.null] * 4
            d[i % 20] = np.array([[i * 1000, 1]])

            out = inst(d)
            if i == 40:
                self.assertEquals(np.sum(out[0] - np.array([[0, 2]])), 0)
            elif i == 60:
                self.assertEquals(np.sum(out[0] - np.array([[20 * 1000, 2]])),
                                  0)
            elif i == 100:
                self.assertEquals(
                    np.sum(out[0] -
                           np.array([[40 * 1000, 2], [60 * 1000, 2]])), 0)
Example #2
0
    def setup(self, opts):
        OperatorDriver.setup(self, opts)
        self.restrict = opts.get('Restrict')
        self.group = opts.get('Group', None)
        self.tz = opts.get('Timezone', core.Timeseries.DEFAULTS['Properties/Timezone'])

        # specialize the input operators
        self.ops = []
        for k, v in opts.iteritems():
            if not k.startswith('Expression'): continue
            self.ops.append(parse_opex(v))
Example #3
0
    def setup(self, opts):
        OperatorDriver.setup(self, opts)
        self.restrict = opts.get('Restrict')
        self.group = opts.get('Group', None)
        self.tz = opts.get('Timezone',
                           core.Timeseries.DEFAULTS['Properties/Timezone'])

        # specialize the input operators
        self.ops = []
        for k, v in opts.iteritems():
            if not k.startswith('Expression'): continue
            self.ops.append(parse_opex(v))
Example #4
0
    def test_mergedsum(self):
        q = "nansum(axis=1) < paste < swindow(first, 20)"
        op = parse_opex(q).ast
        inst = op([{'uuid' : str(uuid.uuid1()), 'Properties/Timezone': 'America/Los_Angeles'},
                   {'uuid' : str(uuid.uuid1()), 'Properties/Timezone': 'America/Los_Angeles'},
                   {'uuid' : str(uuid.uuid1()), 'Properties/Timezone': 'America/Los_Angeles'},
                   {'uuid' : str(uuid.uuid1()), 'Properties/Timezone': 'America/Los_Angeles'}])
        self.assertFalse(inst.block_streaming)
        self.assertEquals(len(inst.outputs), 1)

        for i in [0, 1, 20, 21, 40, 41, 60, 61, 100, 101, 200, 300]:
            d = [operators.null] * 4
            d[i % 20] = np.array([[i * 1000, 1]])

            out = inst(d)
            if i == 40:
                self.assertEquals(np.sum(out[0] - np.array([[0, 2]])), 0)
            elif i == 60:
                self.assertEquals(np.sum(out[0] - np.array([[20 * 1000, 2]])), 0)
            elif i == 100:
                self.assertEquals(np.sum(out[0] - np.array([[40 * 1000, 2], [60 * 1000, 2]])), 0)
Example #5
0
    def process(self, streams_wrapped, op, start=1):
        """ Apply op to streams in streams_wrapped from start to latest"""
        op_a = parse_opex(op.opstr)

        # this will work at least until the year 33658
        d_spec = {'start': start, 'end': 1000000000000000000, 
                                    'limit': [0, 0], 'method': 'data'}
        for stream in streams_wrapped:
            fetch_res = fetch_streamid(stream.uuid)
            stream.streamid = fetch_res[0]
            print("streamid is "  + str(fetch_res[0]))
            stream.subscription_key = fetch_res[1]
            print("sub key is " + str(fetch_res[1]))
        cons = ProcessedDataConsumer(streams_wrapped, op)
        cons.materializer = self
        op_app = OperatorApplicator(op_a, d_spec, cons)
        op_app.DATA_DAYS = 100000000000
        metas = [getattr(stream, 'metadata') for stream in streams_wrapped]
        ids = [[getattr(stream, 'uuid'), getattr(stream,
                                    'streamid') ] for stream in streams_wrapped]
        op_app.start_processing(((True, metas), (True, ids)))