コード例 #1
0
ファイル: operators.py プロジェクト: tarunsmalviya/smap
    def load_crossection(self, *args):
        start = self.start_dt
        end = self.start_dt + self.load_chunk_size
        if end > self.end_dt: end = self.end_dt

        log.msg("loading " + str(self.load_offset) + " " +
                str(start) + ' - ' + str(end))
        start, end  = dtutil.dt2ts(start), \
            dtutil.dt2ts(end)

        d = threads.deferToThread(self.arclient.data_uuid, 
                                  self.load_uids[self.load_offset:
                                                     self.load_offset + 
                                                 self.load_xsec_size],
                                  start, end, 
                                  self.cache)

        d.addCallback(self.load_data, self.load_offset)
        d.addCallback(lambda _: (self._flush(), None))

        self.load_offset += self.load_xsec_size
        if self.load_offset >= len(self.load_uids):
            # pick a new window
            self.start_dt += self.load_chunk_size
            d.addCallback(self.load_time_chunk)
        else:
            d.addCallback(self.load_crossection)
        def err(e):
            print e
        d.addErrback(err)

        return d
コード例 #2
0
ファイル: opc_exe.py プロジェクト: tarunsmalviya/smap
    def _update(self, val):
        print "update using", val
        try:
            point, value, quality, time = val
            if quality != 'Good':
                log.msg("bad quality on point " + point + ": " + quality)
                return

            # parse the timestamp in the timezone of the server
            if self.use_opc_timestamps == 'true':
                ts = dtutil.strptime_tz(time,
                                        self.opc_timefmt,
                                        self.opc_timezone)
                ts = dtutil.dt2ts(ts)
            else:
                ts = dtutil.dt2ts(dtutil.now())

            path = self.make_path(point)
            series = self.get_timeseries(path)
            if series:
                if series['Properties']['ReadingType'] == 'double':
                    series._add(ts, float(value))
                else:
                    series._add(ts, int(value))
        except:
            log.err()
コード例 #3
0
ファイル: example.py プロジェクト: immesys/smap-1
 def load_data(self, st, et):
     st_utc = dtutil.dt2ts(st)
     et_utc = dtutil.dt2ts(et)
     ts = int(st_utc / 120) * 120 # round down to nearest 2-min increment
     while ts <= et_utc:
         self.add('/sensor0', ts, self.counter)
         self.counter += 1
         ts += 120 # 2-min increments
コード例 #4
0
ファイル: example.py プロジェクト: tarunsmalviya/smap
 def load_data(self, st, et):
     st_utc = dtutil.dt2ts(st)
     et_utc = dtutil.dt2ts(et)
     ts = int(st_utc / 120) * 120  # round down to nearest 2-min increment
     while ts <= et_utc:
         self.add('/sensor0', ts, self.counter)
         self.counter += 1
         ts += 120  # 2-min increments
コード例 #5
0
ファイル: opc_auto.py プロジェクト: aslakjohansen/smap
 def _update(self):
     vals = self.opc.read(group="smap-points-group")
     for point, value, quality, time in vals:
         # parse the timestamp in the timezone of the server
         if time is not None:
             ts = dtutil.strptime_tz(time, self.opc_timefmt, self.opc_timezone)
             ts = dtutil.dt2ts(ts)
         else:
             ts = dtutil.now(self.opc_timezone)
             ts = dtutil.dt2ts(ts)
         if self.get_timeseries(self.make_path(point)) and value is not None:
             if isinstance(value, bool): value = int(value)
             self._add(self.make_path(point), ts, float(value))
コード例 #6
0
 def _update(self):
     vals = self.opc.read(group="smap-points-group")
     for point, value, quality, time in vals:
         # parse the timestamp in the timezone of the server
         if time is not None:
             ts = dtutil.strptime_tz(time, self.opc_timefmt,
                                     self.opc_timezone)
             ts = dtutil.dt2ts(ts)
         else:
             ts = dtutil.now(self.opc_timezone)
             ts = dtutil.dt2ts(ts)
         if self.get_timeseries(
                 self.make_path(point)) and value is not None:
             if isinstance(value, bool): value = int(value)
             self._add(self.make_path(point), ts, float(value))
コード例 #7
0
    def test_fill_missing(self):

        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        # check that we fill the end correctly
        op = grouping.GroupByDatetimeField(self.inputs,
                                           arithmetic.first,
                                           field='hour',
                                           width=1,
                                           skip_missing=False)
        now *= 1000
        rv = op(
            operators.DataChunk((now, now + ((self.hours + 5) * 3600 * 1000)),
                                True, True, [self.testdata]))
        self.assertEqual(len(rv[0]), self.hours + 5)
        self.assertEqual(np.sum(np.isnan(rv[0][-5:, 1])), 5)
        self.assertEqual(np.sum(np.isnan(rv[0][:-5, 1])), 0)

        # and the beginning
        op = grouping.GroupByDatetimeField(self.inputs,
                                           arithmetic.first,
                                           field='hour',
                                           width=1,
                                           skip_missing=False)
        rv = op(
            operators.DataChunk(
                (now - (5 * 3600 * 1000), now + ((self.hours) * 3600 * 1000)),
                True, True, [self.testdata]))
        self.assertEqual(len(rv[0]), self.hours + 5)
        self.assertEqual(np.sum(np.isnan(rv[0][:5, 1])), 5)
        self.assertEqual(np.sum(np.isnan(rv[0][5:, 1])), 0)
コード例 #8
0
ファイル: opc.py プロジェクト: tarunsmalviya/smap
 def _update(self):
     vals = self.opc.read(group="smap-points-group")
     for point, value, quality, time in vals:
         # parse the timestamp in the timezone of the server
         ts = dtutil.strptime_tz(time, self.opc_timefmt, self.opc_timezone)
         ts = dtutil.dt2ts(ts)
         self._add(self.make_path(point), ts, value)
コード例 #9
0
ファイル: opc.py プロジェクト: rraabb/smap
 def _update(self):
     vals = self.opc.read(group="smap-points-group")
     for point, value, quality, time in vals:
         # parse the timestamp in the timezone of the server
         ts = dtutil.strptime_tz(time, self.opc_timefmt, self.opc_timezone)
         ts = dtutil.dt2ts(ts)
         self._add(self.make_path(point), ts, value)
コード例 #10
0
 def read(self):
     object_ = {}
     print 'read running'
     try:
         #get the text from the ur
         wa = urllib2.urlopen(
             'http://transmission.bpa.gov/business/operations/wind/baltwg.txt'
         )
         data = [
             line for line in wa.readlines()[7:] if len(line.split()) > 3
         ]
         #parse most recent data
         rawTime = " ".join(data[-1].split()[:2])
         currentTime = int(
             dtutil.dt2ts(
                 dtutil.strptime_tz(rawTime, "%m/%d/%Y %H:%M",
                                    'US/Pacific')))
         object_["Wind"] = data[-1].split()[3]
         object_["Hydro"] = data[-1].split()[4]
         object_["Thermal"] = data[-1].split()[5]
         object_["Load"] = data[-1].split()[2]
     except Exception as e:
         logging.exception(type(e))
         print e
     else:
         if currentTime != self.previousTime:
             self.w.add(currentTime, int(object_["Wind"]))
             self.h.add(currentTime, int(object_["Hydro"]))
             self.t.add(currentTime, int(object_["Thermal"]))
             self.l.add(currentTime, int(object_["Load"]))
             self.previousTime = currentTime
         wa.close()
コード例 #11
0
	def read(self):
		object_ = {}
		print 'read running'
		try:
			#get the text from the ur
			wa = urllib2.urlopen('http://transmission.bpa.gov/business/operations/wind/baltwg.txt')
			data = [line for line in wa.readlines()[7:] if len(line.split()) > 3]
			#parse most recent data
			rawTime = " ".join(data[-1].split()[:2])
			currentTime = int(dtutil.dt2ts(dtutil.strptime_tz(rawTime,"%m/%d/%Y %H:%M",'US/Pacific')))
			object_["Wind"] = data[-1].split()[3]
			object_["Hydro"] = data[-1].split()[4]
			object_["Thermal"] = data[-1].split()[5]
			object_["Load"] = data[-1].split()[2]
		except Exception as e:
			logging.exception(type(e))
			print e
		else:
			if currentTime != self.previousTime:
				self.w.add(currentTime,int(object_["Wind"]))
				self.h.add(currentTime,int(object_["Hydro"]))
				self.t.add(currentTime,int(object_["Thermal"]))
				self.l.add(currentTime,int(object_["Load"]))
				self.previousTime = currentTime
			wa.close()
コード例 #12
0
    def test_inclusive(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        #         startshape = self.testdata.shape
        #         startdata = np.copy(self.testdata)

        #         op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
        #         rv = op([self.testdata[:30, :]])
        #         self.assertEquals(rv[0].shape, (24, 2))
        #         # check for mutations
        #         self.assertEquals(self.testdata.shape, startshape)
        #         self.assertEquals(np.sum(startdata - self.testdata), 0)

        op2 = grouping.GroupByDatetimeField(self.inputs,
                                            oputils.NullOperator,
                                            field='day',
                                            inclusive=(True, True),
                                            snap_times=False)
        rv = op2([self.testdata[0:30, :]])
        self.assertEquals(rv[0].shape, (25, 2))
        self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
        self.assertEquals(rv[0][24, 0], self.testdata[24, 0])
コード例 #13
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def test_fill_missing(self):

        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)


        # check that we fill the end correctly
        op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
                                           field='hour',
                                           width=1,
                                           skip_empty=False)
        now *= 1000
        rv = op(operators.DataChunk((now,
                                     now + ((self.hours + 5) * 3600 * 1000)), 
                                    True, True, 
                                    [self.testdata]))
        self.assertEqual(len(rv[0]), self.hours + 5)
        self.assertEqual(np.sum(np.isnan(rv[0][-5:, 1])), 5)
        self.assertEqual(np.sum(np.isnan(rv[0][:-5, 1])), 0)


        # and the beginning
        op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
                                           field='hour',
                                           width=1,
                                           skip_empty=False)
        rv = op(operators.DataChunk((now - (5 * 3600 * 1000),
                                     now + ((self.hours) * 3600 * 1000)), 
                                    True, True, 
                                    [self.testdata]))
        self.assertEqual(len(rv[0]), self.hours + 5)
        self.assertEqual(np.sum(np.isnan(rv[0][:5, 1])), 5)
        self.assertEqual(np.sum(np.isnan(rv[0][5:, 1])), 0)
コード例 #14
0
ファイル: queryparse.py プロジェクト: tj800x/bemoss_os_1.2
def p_timeref(t):
    """timeref : abstime
               | abstime reltime"""
    t[0] = t[1]
    if len(t) == 2:
        ref = t[1]
    else:
        ref = t[1] + t[2]
    t[0] = dtutil.dt2ts(ref) * 1000
コード例 #15
0
ファイル: queryparse.py プロジェクト: immesys/smap-1
def p_timeref(t):
    """timeref : abstime
               | abstime reltime"""
    t[0] = t[1]
    if len(t) == 2:
        ref = t[1]
    else:
        ref = t[1] + t[2]
    t[0] = dtutil.dt2ts(ref) * 1000
コード例 #16
0
ファイル: test_group.py プロジェクト: ahaas/smap
 def test_snap_times(self):
     now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
     now = dtutil.dt2ts(now)
     self.setUp(now)
     
     op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first, 
                                        field='day', 
                                        snap_times=True)
     rv = op([self.testdata[10:30]])
     self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
コード例 #17
0
ファイル: test_group.py プロジェクト: ahaas/smap
 def test_offset(self):
     now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
     now = dtutil.dt2ts(now)
     self.setUp(now)
     
     op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
     for i in xrange(0, 24):
         rv = op([self.testdata[i:25+i, :]])
         self.assertEquals(rv[0].shape, (24 - i, 2))
         op.reset()
コード例 #18
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def setUp(self):
        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)

        self.testdata = np.ones((self.hours, 2))
        for i in xrange(0, self.hours):
            self.testdata[i, :] = i
            
        self.testdata[:, 0] *= 3600 
        self.testdata[:, 0] += now 
        self.ma = grouping.MaskedDTList(self.testdata[:, 0], dtutil.gettz("America/Los_Angeles"))
        self.width = datetime.timedelta(days=1)
コード例 #19
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def test_slide(self):
        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
                                           field='hour',
                                           width=4,
                                           slide=2)
        rv = op([self.testdata])
        self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[:-2:2, 0]), 0)
        self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:-2:2, 1]), 0)
コード例 #20
0
    def test_snap_times(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs,
                                           arithmetic.first,
                                           field='day',
                                           snap_times=True)
        rv = op([self.testdata[10:30]])
        self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
コード例 #21
0
ファイル: ted.py プロジェクト: SoftwareDefinedBuildings/BSE
 def process(self, doc):
     doc = bs(doc)
     now = doc.livedata.gatewaytime
     now = dtutil.strptime_tz("%s %s %s %s %s %s" % (now.month.contents[0], 
                                                     now.day.contents[0], 
                                                     now.year.contents[0], 
                                                     now.hour.contents[0], 
                                                     now.minute.contents[0], 
                                                     now.maxsecond.contents[0]),
                              "%m %d %y %H %M %S", tzstr=self.timezone)
     now = dtutil.dt2ts(now)
     self.add('/voltage', now, int(doc.livedata.voltage.total.voltagenow.contents[0]))
     self.add('/real_power', now, int(doc.livedata.power.total.powernow.contents[0]))
     self.add('/apparent_power', now, int(doc.livedata.power.total.kva.contents[0]))
コード例 #22
0
    def test_offset(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs,
                                           oputils.NullOperator,
                                           field='day')
        for i in xrange(0, 24):
            rv = op([self.testdata[i:25 + i, :]])
            self.assertEquals(rv[0].shape, (24 - i, 2))
            op.reset()
コード例 #23
0
    def setUp(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)

        self.testdata = np.ones((self.hours, 2))
        for i in xrange(0, self.hours):
            self.testdata[i, :] = i

        self.testdata[:, 0] *= 3600
        self.testdata[:, 0] += now
        self.ma = grouping.MaskedDTList(self.testdata[:, 0],
                                        dtutil.gettz("America/Los_Angeles"))
        self.width = datetime.timedelta(days=1)
コード例 #24
0
    def test_slide(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs,
                                           arithmetic.first,
                                           field='hour',
                                           width=4,
                                           slide=2)
        rv = op([self.testdata])
        self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[:-2:2, 0]), 0)
        self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:-2:2, 1]), 0)
コード例 #25
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def test_flush(self):
        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
                                           field='hour',
                                           width=1)
        rv = op(operators.DataChunk((now * 1000, 
                                     now * 1000 + (self.hours * 3600 * 1000)), 
                                    True, True, 
                                    [self.testdata]))

        # if we don't properly flush the last hour, we should only get hours - 1 results
        self.assertEquals((rv[0][-1, 0] - (now *  1000)) / (3600 * 1000), self.hours - 1)
コード例 #26
0
ファイル: resamplr.py プロジェクト: tarunsmalviya/smap
def _meter_sample(
        data,
        slop=datetime.timedelta(minutes=30),
        bin_equal=_day_bin_equal,
        # state parameters
        prev=None,  # start of meter region
        trianglestart=None,  # last reset
        accum=0,  # accumulated energy in window
        prev_data=None):  # last point we saw
    rv = []
    if len(data) and (not prev or not trianglestart):
        prev = data[0]
        prev_data = data[0]
        trianglestart = data[0]
        accum = 0
        start = 1
    else:
        start = 0

    for i, v in enumerate(data[start:]):
        if v[1] < trianglestart[1]:
            # if we roll over, add in the accumulated sum through now
            accum += (prev_data[1] - trianglestart[1])
            trianglestart = v

        if bin_equal(prev[0], v[0]):
            # continue if we're still in the same bin
            prev_data = v
            continue
        elif bin_equal(prev[0], v[0] - slop) and \
                not bin_equal(prev[0] - slop, prev[0]):
            # otherwise produce a record if the endpoints are close to

            accum += prev_data[1] - trianglestart[1]
            print "add", prev[0], accum, trianglestart[0]
            rv.append((dtutil.dt2ts(prev[0]), accum))
            prev_data = v

        prev = v
        trianglestart = v
        accum = 0

    return np.array(rv), {
        'prev': prev,
        'prev_data': prev_data,
        'trianglestart': trianglestart,
        'accum': accum
    }
コード例 #27
0
 def OnMultipleItemsChanged(self, sender, args):
     for v in args.ArgsArray:
         driver = OPC_DRIVERS[v.State]
         if v.Exception:
             continue
         print v.ItemDescriptor.ItemId,
         print v.Vtq.Timestamp, v.Vtq.Value
         ts = dtutil.strptime_tz(str(v.Vtq.Timestamp),
                                 "%m/%d/%y %H:%M:%S",
                                 tzstr=driver.opc_timezone)
         path = driver.make_path(v.ItemDescriptor.ItemId)
         try:
             driver._add(driver.make_path(v.ItemDescriptor.ItemId),
                         int(dtutil.dt2ts(ts)), v.Vtq.Value)
         except Exception, e:
             log.err("Error adding data: " + str(e))
コード例 #28
0
def _meter_sample(data, 
                  slop = datetime.timedelta(minutes=30),
                  bin_equal=_day_bin_equal,
                  # state parameters
                  prev=None,    # start of meter region
                  trianglestart=None, # last reset
                  accum=0,            # accumulated energy in window
                  prev_data=None):    # last point we saw
    rv = []
    if len(data) and (not prev or not trianglestart):
        prev = data[0]
        prev_data = data[0]
        trianglestart = data[0]
        accum = 0
        start = 1
    else:
        start = 0
    
    for i, v in enumerate(data[start:]):
        if v[1] < trianglestart[1]:
            # if we roll over, add in the accumulated sum through now
            accum += (prev_data[1] - trianglestart[1])
            trianglestart = v

        if bin_equal(prev[0], v[0]):
            # continue if we're still in the same bin
            prev_data = v
            continue
        elif bin_equal(prev[0], v[0] - slop) and \
                not bin_equal(prev[0] - slop, prev[0]):
            # otherwise produce a record if the endpoints are close to 

            accum += prev_data[1] - trianglestart[1]
            print "add", prev[0], accum, trianglestart[0]
            rv.append((dtutil.dt2ts(prev[0]), accum))
            prev_data = v

        prev = v
        trianglestart = v
        accum = 0

    return np.array(rv), {
        'prev' : prev,
        'prev_data' : prev_data,
        'trianglestart' : trianglestart,
        'accum' : accum
        }
コード例 #29
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def test_oneatatime(self):
        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
        for i in xrange(0, 24):
            rv = op([self.testdata[i:i+1, :]])
            self.assertEquals(rv[0].shape, operators.null.shape)

        rv = op([self.testdata[24:25, :]])

        self.assertEquals(rv[0].shape, (24, 2))
        # make sure we snapped
        self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[0, 0]), 0)
        # and got back the right data
        self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:24, 1]), 0)
コード例 #30
0
 def process(self, doc):
     doc = bs(doc)
     now = doc.livedata.gatewaytime
     now = dtutil.strptime_tz(
         "%s %s %s %s %s %s" %
         (now.month.contents[0], now.day.contents[0], now.year.contents[0],
          now.hour.contents[0], now.minute.contents[0],
          now.maxsecond.contents[0]),
         "%m %d %y %H %M %S",
         tzstr=self.timezone)
     now = dtutil.dt2ts(now)
     self.add('/voltage', now,
              int(doc.livedata.voltage.total.voltagenow.contents[0]))
     self.add('/real_power', now,
              int(doc.livedata.power.total.powernow.contents[0]))
     self.add('/apparent_power', now,
              int(doc.livedata.power.total.kva.contents[0]))
コード例 #31
0
 def OnMultipleItemsChanged(self, sender, args):
     for v in args.ArgsArray:
         driver = OPC_DRIVERS[v.State]
         if v.Exception:
             continue
         print v.ItemDescriptor.ItemId,
         print v.Vtq.Timestamp, v.Vtq.Value
         ts = dtutil.strptime_tz(str(v.Vtq.Timestamp),
                                 "%m/%d/%y %H:%M:%S",
                                 tzstr=driver.opc_timezone)
         path = driver.make_path(v.ItemDescriptor.ItemId)
         try:
             driver._add(driver.make_path(v.ItemDescriptor.ItemId),
                         int(dtutil.dt2ts(ts)),
                         v.Vtq.Value)
         except Exception, e:
             log.err("Error adding data: " + str(e))
コード例 #32
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def test_increment(self):
        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        for incr in [2, 4, 6, 8, 12, 24]:
            op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first, 
                                               field='hour', 
                                               width=incr)
            rv = op([self.testdata[:25, :]])
            # check the shape
            self.assertEquals(len(rv[0]), 24 / incr)
            for i in xrange(0, 24/incr):
                # the timestamps
                self.assertEquals(rv[0][i, 0], self.testdata[i * incr, 0])
                # and the values
                self.assertEquals(rv[0][i, 1], i * incr)
            del op
コード例 #33
0
    def test_flush(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs,
                                           arithmetic.first,
                                           field='hour',
                                           width=1)
        rv = op(
            operators.DataChunk(
                (now * 1000, now * 1000 + (self.hours * 3600 * 1000)), True,
                True, [self.testdata]))

        # if we don't properly flush the last hour, we should only get hours - 1 results
        self.assertEquals((rv[0][-1, 0] - (now * 1000)) / (3600 * 1000),
                          self.hours - 1)
コード例 #34
0
    def process_one(self, data, tz, 
                    prev=None,
                    prev_datetimes=None):
        times, values = data[:,0], data[:,1]

        if (prev is not None):
            times = np.append(prev_datetimes, times)
            values = np.append(prev, values)
            st = self.snapper(dtutil.ts2dt(prev_datetimes[-1] / 1000))
        else:
            st = self.snapper(dtutil.ts2dt(times[0] / 1000))
        st = dtutil.dt2ts(st) * 1000 + self.width
        et = int(times[-1])
        mesh = np.arange(st, et, self.width)
        
        if (self.max_time_delta): 
            gaps = self.detect_gaps(times)
            remove = np.array([False] * len(mesh))
            for gap in gaps:
                gt = np.greater(mesh, gap[0])
                lt = np.less(mesh, gap[1])
                this_gap = np.logical_and(gt, lt)
                remove = np.logical_or(remove, this_gap)
            remove_inds = np.nonzero(remove)[0]
            mesh = np.delete(mesh, remove_inds)
        
        if (self.method == 'linear'):
            outvals = np.interp(mesh, times, values)
            prev = np.array([values[-1]])
            prev_datetimes = np.array([times[-1]])
        elif (self.method == 'spline'):
            s = UnivariateSpline(times, values, s=0) 
            outvals = s(mesh) 
            # 10 points = empirical
            prev = np.array(values[-10:])
            prev_datetimes = np.array(times[-10:])
        output = np.vstack((mesh,outvals)).T
        state = { 'prev': prev,
                  'prev_datetimes': prev_datetimes,
                }
        
        return output, state
コード例 #35
0
    def process_one(self, data, tz, prev=None, prev_datetimes=None):
        times, values = data[:, 0], data[:, 1]

        if (prev is not None):
            times = np.append(prev_datetimes, times)
            values = np.append(prev, values)
            st = self.snapper(dtutil.ts2dt(prev_datetimes[-1] / 1000))
        else:
            st = self.snapper(dtutil.ts2dt(times[0] / 1000))
        st = dtutil.dt2ts(st) * 1000 + self.width
        et = int(times[-1])
        mesh = np.arange(st, et, self.width)

        if (self.max_time_delta):
            gaps = self.detect_gaps(times)
            remove = np.array([False] * len(mesh))
            for gap in gaps:
                gt = np.greater(mesh, gap[0])
                lt = np.less(mesh, gap[1])
                this_gap = np.logical_and(gt, lt)
                remove = np.logical_or(remove, this_gap)
            remove_inds = np.nonzero(remove)[0]
            mesh = np.delete(mesh, remove_inds)

        if (self.method == 'linear'):
            outvals = np.interp(mesh, times, values)
            prev = np.array([values[-1]])
            prev_datetimes = np.array([times[-1]])
        elif (self.method == 'spline'):
            s = UnivariateSpline(times, values, s=0)
            outvals = s(mesh)
            # 10 points = empirical
            prev = np.array(values[-10:])
            prev_datetimes = np.array(times[-10:])
        output = np.vstack((mesh, outvals)).T
        state = {
            'prev': prev,
            'prev_datetimes': prev_datetimes,
        }

        return output, state
コード例 #36
0
    def test_increment(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        for incr in [2, 4, 6, 8, 12, 24]:
            op = grouping.GroupByDatetimeField(self.inputs,
                                               arithmetic.first,
                                               field='hour',
                                               width=incr)
            rv = op([self.testdata[:25, :]])
            # check the shape
            self.assertEquals(len(rv[0]), 24 / incr)
            for i in xrange(0, 24 / incr):
                # the timestamps
                self.assertEquals(rv[0][i, 0], self.testdata[i * incr, 0])
                # and the values
                self.assertEquals(rv[0][i, 1], i * incr)
            del op
コード例 #37
0
    def test_oneatatime(self):
        now = dtutil.strptime_tz("1 1 2000 0",
                                 "%m %d %Y %H",
                                 tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

        op = grouping.GroupByDatetimeField(self.inputs,
                                           oputils.NullOperator,
                                           field='day')
        for i in xrange(0, 24):
            rv = op([self.testdata[i:i + 1, :]])
            self.assertEquals(rv[0].shape, operators.null.shape)

        rv = op([self.testdata[24:25, :]])

        self.assertEquals(rv[0].shape, (24, 2))
        # make sure we snapped
        self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[0, 0]), 0)
        # and got back the right data
        self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:24, 1]), 0)
コード例 #38
0
    def read(self):
        for retry_time in [0, 30, 5 * 60]:
            time.sleep(retry_time)
            try:
                print "Reading"
                data = urllib2.urlopen(self.url, timeout=30).read()
                times = {}

                b = BeautifulSoup.BeautifulSoup(data)

                # Parse time blocks
                data = b.find('data')
                for time_block in data.findAll('time-layout'):
                    key = time_block.find('layout-key').contents[0]
                    time_list = []
                    for time_tag in time_block.findAll('start-valid-time'):
                        #dt = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S%z')
                        dt = dateutil.parser.parse(time_tag.string)
                        time_list.append(dtutil.dt2ts(dt))
                    times[key] = time_list

                # For each value block find referenced time block and add readings
                for data_block in data.find('parameters').findAll(
                        recursive=False):
                    key = data_block['time-layout']
                    # Find the element being returned
                    for (name, e) in self.element_map.items():
                        if e.tag_name == data_block.name and e.type_name == data_block[
                                'type']:
                            # Element found
                            value = []
                            for v in data_block.findAll('value'):
                                value.append(float(v.string))

                            for t, v in zip(times[key], value):
                                self.add('/' + name, int(t), v)
                            break
                return
            except Exception, e:
                print e
コード例 #39
0
ファイル: test_group.py プロジェクト: ahaas/smap
    def test_inclusive(self):
        now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
        now = dtutil.dt2ts(now)
        self.setUp(now)

#         startshape = self.testdata.shape
#         startdata = np.copy(self.testdata)

#         op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
#         rv = op([self.testdata[:30, :]])
#         self.assertEquals(rv[0].shape, (24, 2))
#         # check for mutations
#         self.assertEquals(self.testdata.shape, startshape)
#         self.assertEquals(np.sum(startdata - self.testdata), 0)

        op2 = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, 
                                            field='day', inclusive=(True, True),
                                            snap_times=False)
        rv = op2([self.testdata[0:30, :]])
        self.assertEquals(rv[0].shape, (25, 2))
        self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
        self.assertEquals(rv[0][24, 0], self.testdata[24, 0])
コード例 #40
0
  def read(self):
    for retry_time in [0, 30, 5*60]:
      time.sleep(retry_time)
      try:
        print "Reading"
        data = urllib2.urlopen(self.url, timeout = 30).read()
        times = {}

        b = BeautifulSoup.BeautifulSoup(data)
        
        # Parse time blocks
        data = b.find('data')
        for time_block in data.findAll('time-layout'):
          key = time_block.find('layout-key').contents[0]
          time_list = []
          for time_tag in time_block.findAll('start-valid-time'):
            #dt = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S%z')
            dt = dateutil.parser.parse(time_tag.string)
            time_list.append(dtutil.dt2ts(dt))
          times[key] = time_list

        # For each value block find referenced time block and add readings
        for data_block in data.find('parameters').findAll(recursive=False):
          key = data_block['time-layout']
          # Find the element being returned
          for (name, e) in self.element_map.items():
            if e.tag_name == data_block.name and e.type_name == data_block['type']:
              # Element found
              value = []
              for v in data_block.findAll('value'):
                value.append(float(v.string))
              
              for t,v in zip(times[key], value):
                self.add('/'+name, int(t), v)
              break
        return
      except Exception, e:
        print e
コード例 #41
0
ファイル: bmo.py プロジェクト: yisea123/smap-data
    def process(self):
        readcnt = 0
        data = []

        if self.reader == None: 
            return data

        try:
            for r in self.reader:
                ts = dtutil.strptime_tz(r[0], TIMEFMT, tzstr='UTC')
                if ts > self.push_hist:
                    self.push_hist = ts
                ts = dtutil.dt2ts(ts)

                data.append((ts, zip(self.field_map, r)))

                readcnt += 1
                if readcnt > 100:
                    return data
        except Exception, e:
            self.fp.close()
            self.reader = None
            raise e
コード例 #42
0
ファイル: bmo_no403.py プロジェクト: tarunsmalviya/smap
    def process(self, body):
        reader = csv.reader(StringIO.StringIO(body), dialect='excel-tab')
        header = reader.next()
        if len(header) == 0:
            print "Warning: no data from", self.url
            raise core.SmapException("no data!")
        try:
            self.field_map, self.map = make_field_idxs(self.meter_type, header, 
                                                       location=self.location)
        except:
            traceback.print_exc()

        if not self.added:
            self.added = True
            for channel in self.map['sensors'] + self.map['meters']:
                try:
                    self.add_timeseries('/%s/%s' % channel[2:4], channel[4], data_type='double')
                    self.set_metadata('/%s/%s' % channel[2:4], {
                            'Extra/ChannelName' : re.sub('\(.*\)', '', channel[0]).strip(),
                            })
                    
                except:
                    traceback.print_exc()

        # add all the values
        for r in reader:
            ts = dtutil.strptime_tz(r[0], TIMEFMT, tzstr='UTC')
            if ts > self.push_hist:
                self.push_hist = ts
            ts = dtutil.dt2ts(ts)

            for descr, val in zip(self.field_map, r):
                if descr == None: continue
                try:
                    self._add('/' + '/'.join(descr), ts, float(val))
                except ValueError:
                    pass
コード例 #43
0
ファイル: energy_plot.py プロジェクト: Thunder1989/SDB
bldg = bldg_dict[int(num)]
# start = raw_input("start time (\"%m-%d-%Y %H:%M\" or "-d" for default): ")
# end = raw_input("end time (\"%m-%d-%Y %H:%M\") or "-d" for default: ")
start = "10-21-2013 00:00"
end = "10-27-2013 23:59"
# get the outside air temperature during the period specified
# get_temp(start, end)

if start == '-d':
	print "computing the IMFs in %s from 7 days backwards till now..." %bldg
else:
	print "computing the IMFs in %s from %s to %s..." %(bldg, start, end)
if end == '-d':
	end = int(time.time()*1000)
else:
	end = dtutil.dt2ts(dtutil.strptime_tz("%s" %end, "%m-%d-%Y %H:%M")) * 1000
if start == '-d':
	start = end - 7*24*60*60*1000
else:
	start = dtutil.dt2ts(dtutil.strptime_tz("%s" %start, "%m-%d-%Y %H:%M")) * 1000
applySum = "apply nansum(axis=1) < paste < window(first, field='minute', width=15) < units to data in (%f, %f) limit -1 streamlimit 10000 \
	where (Metadata/Extra/System = 'total' or Metadata/Extra/System = 'electric') \
	and (Properties/UnitofMeasure = 'kW' or Properties/UnitofMeasure = 'Watts' or Properties/UnitofMeasure = 'W') \
	and Metadata/Location/Building like '%s%%' and not Metadata/Extra/Operator like 'sum%%' \
	and not Path like '%%demand' and not Path like '/Cory_Hall/Electric_5A7/ABC/real_power' and not Path like '/Cory_Hall/Electric_5B7/ABC/real_power'" \
	%(start, end, bldg)

result = c.query(applySum) #the result is a list
reading = result[0]['Readings']

# output readings to file
コード例 #44
0
    def process_one(self, data, op, tz,
                    prev=None,
                    prev_datetimes=None,
                    first=False, last=False,
                    region=(None, None)):
        # print "PRCESSING"
        tic = time.time()
        if prev == None:
            prev = np.copy(data)
            prev_datetimes = MaskedDTList(prev[:, 0] / 1000, tz)
        else:
            prev = np.vstack((prev, data))
            prev_datetimes.extend(data[:, 0] / 1000)
        
        assert len(prev_datetimes) == len(prev)
        output = [null] * len(op.outputs)
        # output = [null] * len(util.flatten(map(operator.attrgetter("outputs"), ops)))
        # print output

        if len(prev_datetimes) == 0:
            return output, {
                'prev': prev,
                'prev_datetimes': prev_datetimes,
                }

        # we might want to produce readings before the first data point
        if first and region[0]:
            bin_start = self.snapper(dtutil.ts2dt(region[0] / 1000))
        else:
            bin_start = self.snapper(prev_datetimes[0])
        truncate_to = 0

        while True:

            if last:
                if not region[1] and truncate_to == len(prev_datetimes):
                    break
                if region[1] and region[1] <= dtutil.dt2ts(bin_start) * 1000:
                    break

            bin_end = bin_start + self.bin_slide

            # perform a binary search to find the next window boundary
            bin_start_idx = bisect.bisect_left(prev_datetimes, bin_start) 
            bin_end_idx = bisect.bisect_left(prev_datetimes, bin_end)
            truncate_to = bin_start_idx

            # ignore bins which aren't full
            if bin_end_idx == len(prev_datetimes) and not last:
                break

            # skip empty bins
            if bin_start_idx == bin_end_idx:
                # maybe we were supposed to produce output even if
                # there's no data in the bin
                if not self.skip_empty:
                    t = dtutil.dt2ts(bin_start) * 1000
                    output = extend(output, 
                                    [np.array([[t, np.nan]])])

                bin_start += self.bin_slide
                continue

            if (bin_end_idx < len(prev_datetimes) and 
                self.comparator(bin_start, prev_datetimes[bin_end_idx])):
                take_end = bin_end_idx + 1
            else:
                take_end = bin_end_idx

            opdata = op([prev[bin_start_idx:take_end, :]])
            
            # snap the times to the beginning of the
            # window, if we were asked to.  do this here
            # so we can avoid passing datetimes around,
            # and deal with the common case where this is
            # what ya want.
            if self.snap_times:
                t = dtutil.dt2ts(bin_start)
                for j in xrange(0, len(opdata)):
                    opdata[j][:, 0] = t * 1000
            output = extend(output, opdata)

            bin_start += self.bin_slide

        toc = time.time()
#         print("dt processing took %0.05f: %i/%i converted" %  \
#                   (toc-tic,
#                    prev_datetimes.conversions,
#                    len(prev_datetimes)))

        prev_datetimes.truncate(truncate_to)
        prev = prev[truncate_to:]

        return output, {
            'prev': prev,
            'prev_datetimes': prev_datetimes,
            }
コード例 #45
0
ファイル: keti_fetcher.py プロジェクト: Thunder1989/SDB
from smap.archiver.client import SmapClient
from smap.contrib import dtutil

from matplotlib import pyplot
from matplotlib import dates
import os
# make a client
c = SmapClient("http://new.openbms.org/backend")

# start and end values are Unix timestamps
t_start = "6-12-2013 8:00"
t_end = "6-19-2013 8:00"
start = 1000*dtutil.dt2ts(dtutil.strptime_tz(t_start, "%m-%d-%Y %H:%M"))
end   = 1000*dtutil.dt2ts(dtutil.strptime_tz(t_end, "%m-%d-%Y %H:%M"))

stnc = "select distinct Metadata/Location/RoomNumber where Metadata/SourceName='KETI Motes'"
roomlist = c.query(stnc) #the result is a list

#roomlist = roomlist[16:]
#roomlist = ['621A','621B','621C','621D','621E']
for room in roomlist:
	print "==========Fetching streams in Room %s=========="%room
	stnc = "select Path where Metadata/Location/RoomNumber='%s' and not Path ~ '.*pir.*'" %room
	streams = c.query(stnc)
	if len(streams)>0:
		# print "----%d streams in Room %s----"%(len(streams), room)

		for s in streams:
			# fetch the metadata of path wanted
			tags = c.tags("Path='%s'"%s['Path'])
コード例 #46
0
ファイル: dataclient.py プロジェクト: yshao/weathergit
 def get_data(self,uuid,start,end,limit=-1):
     # print start
     # print end
     startTime = dtutil.dt2ts(dtutil.strptime_tz(start, "%m/%d/%Y %H:%M:%S %p" ))
     endTime   = dtutil.dt2ts(dtutil.strptime_tz(end, "%m/%d/%Y %H:%M:%S %p"))
     return self.c.data_uuid(uuid, startTime, endTime,True,limit)
コード例 #47
0
 def parse_time(self, ts, val):
     if self.timefmt == None:
         return int(val)
     else:
         return dtutil.dt2ts(
             dtutil.strptime_tz(val, self.timefmt, self.timezone))
コード例 #48
0
ファイル: classifier.py プロジェクト: Thunder1989/SDB
search = keywordSearch(lines, "sdh room temp", 2000)
add = keywordSearch(lines, "sdh rm temp", 1000)
for i in water:
	res.remove(i)
for i in stp:
	res.remove(i)
for r in res:
	search_path.append(r['Path'])
for i in add:
	search.append(i)
for i in search:
	search_res.append(i['Path'])

train_start = "6-12-2013 8:00"
train_end = "6-12-2013 8:10"
start1 = 1000*dtutil.dt2ts(dtutil.strptime_tz(train_start, "%m-%d-%Y %H:%M"))
end1 = 1000*dtutil.dt2ts(dtutil.strptime_tz(train_end, "%m-%d-%Y %H:%M"))

test_start = "6-13-2013 10:10"
test_end = "6-13-2013 10:20"
start2 = 1000*dtutil.dt2ts(dtutil.strptime_tz(test_start, "%m-%d-%Y %H:%M"))
end2 = 1000*dtutil.dt2ts(dtutil.strptime_tz(test_end, "%m-%d-%Y %H:%M"))

anomaly_path1 = []
anomaly_path2 = []
train_data = []
test_data = []
search_train = []
# anomaly_train = []
search_test = []
# anomaly_test = []
コード例 #49
0
    def get_readings(self, market, start_date, stop_date):
        readings = {
            'total_price': [],
            'loss': [],
            'energy': [],
            'congestion': []
        }
        print "get_readings", market
        if market == 'DAM':
            q = 'PRC_LMP'
            m = 'DAM'
        elif market == 'HASP':
            q = 'PRC_HASP_LMP'
            m = 'HASP'
        elif market == 'RTM':
            q = 'PRC_INTVL_LMP'
            m = 'RTM'
        else:
            raise Exception("Invalid market: " + market)

        url = 'http://oasis.caiso.com/mrtu-oasis/SingleZip?'
        url += 'queryname=' + q
        url += '&startdate=' + dtutil.strftime_tz(start_date, '%Y%m%d',
                                                  'US/Pacific')
        url += '&enddate=' + dtutil.strftime_tz(stop_date, '%Y%m%d',
                                                'US/Pacific')
        url += '&market_run_id=' + m
        url += '&node=' + self.location

        logging.info("Get url %s" % url)
        h = None
        for d in [5, 20, 60]:
            try:
                h = urllib2.urlopen(url, timeout=50)
                break
            except urllib2.URLError:
                logging.warn("urlopen failed.")
            time.sleep(d)
        if h == None:
            raise Exception("Failed to open url: %s" % url)

        z = zipfile.ZipFile(StringIO.StringIO(h.read()))
        xml = z.read(z.namelist()[0])
        b = BeautifulSoup.BeautifulSoup(xml)

        sec_per_int = int(b.find('m:sec_per_interval').contents[0])

        rows = b.findAll('m:report_data')
        for d in rows:
            res = d.find('m:resource_name').contents[0]
            item = d.find('m:data_item').contents[0]
            day = d.find('m:opr_date').contents[0]
            inter = int(d.find('m:interval_num').contents[0])
            val = float(d.find('m:value').contents[0])

            secs = (inter - 1) * sec_per_int
            dt = dtutil.strptime_tz(
                day, '%Y-%m-%d',
                'US/Pacific') + datetime.timedelta(seconds=secs)
            timestamp = dtutil.dt2ts(dt)

            key = None
            if item == 'LMP_PRC':
                key = 'total_price'
            elif item == 'LMP_LOSS_PRC':
                key = 'loss'
            elif item == 'LMP_ENE_PRC':
                key = 'energy'
            elif item == 'LMP_CONG_PRC':
                key = 'congestion'
            else:
                continue

            readings[key].append((timestamp, val))

        num_readings = len(readings[readings.keys()[0]])
        for k in readings.keys():
            if len(readings[k]) != num_readings:
                raise Exception('Missing readings')

            readings[k] = sorted(readings[k], key=lambda (t, v): t)

        return readings
コード例 #50
0
  def get_readings(self, market, start_date, stop_date): 
    readings = {'total_price': [], 'loss': [], 'energy': [], 'congestion': []}
    print "get_readings", market
    if market == 'DAM':
      q = 'PRC_LMP'
      m = 'DAM'
    elif market == 'HASP':
      q = 'PRC_HASP_LMP'
      m = 'HASP'
    elif market == 'RTM':
      q = 'PRC_INTVL_LMP'
      m = 'RTM'
    else:
      raise Exception("Invalid market: " + market)

    url = 'http://oasis.caiso.com/mrtu-oasis/SingleZip?'
    url += 'queryname=' + q
    url += '&startdate=' + dtutil.strftime_tz(start_date, '%Y%m%d', 'US/Pacific')
    url += '&enddate=' + dtutil.strftime_tz(stop_date, '%Y%m%d', 'US/Pacific')
    url += '&market_run_id=' + m
    url += '&node=' + self.location

    logging.info("Get url %s" % url)
    h = None
    for d in [5, 20, 60]:
      try:
        h = urllib2.urlopen(url, timeout=50)
        break
      except urllib2.URLError:
        logging.warn("urlopen failed.")
      time.sleep(d)
    if h == None:
      raise Exception("Failed to open url: %s" % url)

    z = zipfile.ZipFile(StringIO.StringIO(h.read()))
    xml = z.read(z.namelist()[0])
    b = BeautifulSoup.BeautifulSoup(xml)

    sec_per_int = int( b.find('m:sec_per_interval').contents[0] )

    rows = b.findAll('m:report_data')
    for d in rows:
      res = d.find('m:resource_name').contents[0]
      item = d.find('m:data_item').contents[0]
      day = d.find('m:opr_date').contents[0]
      inter = int( d.find('m:interval_num').contents[0] )
      val = float( d.find('m:value').contents[0] )

      secs = (inter - 1) * sec_per_int
      dt = dtutil.strptime_tz(day, '%Y-%m-%d', 'US/Pacific') + datetime.timedelta(seconds=secs)
      timestamp = dtutil.dt2ts(dt)

      key = None
      if item == 'LMP_PRC':
        key = 'total_price'
      elif item == 'LMP_LOSS_PRC':
        key = 'loss'
      elif item == 'LMP_ENE_PRC':
        key = 'energy'
      elif item == 'LMP_CONG_PRC':
        key = 'congestion'
      else:
        continue

      readings[key].append((timestamp, val))


    num_readings = len(readings[readings.keys()[0]])
    for k in readings.keys():
      if len(readings[k]) != num_readings:
        raise Exception('Missing readings')

      readings[k] = sorted(readings[k], key=lambda (t, v): t)

    return readings
コード例 #51
0
    def process_one(self,
                    data,
                    op,
                    tz,
                    prev=None,
                    prev_datetimes=None,
                    first=False,
                    last=False,
                    region=(None, None)):
        # print "PRCESSING"
        tic = time.time()
        if prev == None:
            prev = np.copy(data)
            prev_datetimes = MaskedDTList(prev[:, 0] / 1000, tz)
        else:
            prev = np.vstack((prev, data))
            prev_datetimes.extend(data[:, 0] / 1000)

        assert len(prev_datetimes) == len(prev)
        output = [null] * len(op.outputs)

        if len(prev_datetimes) == 0:
            return output, {
                'prev': prev,
                'prev_datetimes': prev_datetimes,
            }

        # we might want to produce readings before the first data point
        if first and region[0]:
            bin_start = self.snapper(dtutil.ts2dt(region[0] / 1000))
        else:
            bin_start = self.snapper(prev_datetimes[0])
        truncate_to = 0

        while True:

            if last:
                if not region[1] and truncate_to == len(prev_datetimes):
                    break
                if region[1] and region[1] <= dtutil.dt2ts(bin_start) * 1000:
                    break

            bin_end = bin_start + self.bin_slide

            # perform a binary search to find the next window boundary
            bin_start_idx = bisect.bisect_left(prev_datetimes, bin_start)
            bin_end_idx = bisect.bisect_left(prev_datetimes, bin_end)
            truncate_to = bin_start_idx

            # ignore bins which aren't full
            if (bin_end_idx == len(prev_datetimes) and not last
                    and not (region[1]
                             and dtutil.dt2ts(bin_end) * 1000 <= region[1])):
                break

            # skip empty bins
            if bin_start_idx == bin_end_idx:
                # maybe we were supposed to produce output even if
                # there's no data in the bin
                if not self.skip_empty:
                    t = dtutil.dt2ts(bin_start) * 1000
                    output = extend(output, [np.array([[t, np.nan]])])

                bin_start += self.bin_slide
                continue

            if (bin_end_idx < len(prev_datetimes) and self.comparator(
                    bin_start, prev_datetimes[bin_end_idx])):
                take_end = bin_end_idx + 1
            else:
                take_end = bin_end_idx

            opdata = op([prev[bin_start_idx:take_end, :]])

            # snap the times to the beginning of the
            # window, if we were asked to.  do this here
            # so we can avoid passing datetimes around,
            # and deal with the common case where this is
            # what ya want.
            if self.snap_times:
                t = dtutil.dt2ts(bin_start)
                for j in xrange(0, len(opdata)):
                    opdata[j][:, 0] = t * 1000
            output = extend(output, opdata)

            bin_start += self.bin_slide

        toc = time.time()
        #         print("dt processing took %0.05f: %i/%i converted" %  \
        #                   (toc-tic,
        #                    prev_datetimes.conversions,
        #                    len(prev_datetimes)))

        prev_datetimes.truncate(truncate_to)
        prev = prev[truncate_to:]

        return output, {
            'prev': prev,
            'prev_datetimes': prev_datetimes,
        }
コード例 #52
0
locating the streams using a metadata query.

@author Stephen Dawson-Haggerty <*****@*****.**>
"""

from smap.archiver.client import SmapClient
from smap.contrib import dtutil

from matplotlib import pyplot
from matplotlib import dates

# make a client
c = SmapClient("http://www.openbms.org/backend")

# start and end values are Unix timestamps
start = dtutil.dt2ts(dtutil.strptime_tz("1-1-2013", "%m-%d-%Y"))
end   = dtutil.dt2ts(dtutil.strptime_tz("1-2-2013", "%m-%d-%Y"))

# download the data and metadata
tags = c.tags("Metadata/Extra/Type = 'oat'")
uuids, data = c.data("Metadata/Extra/Type = 'oat'", start, end)

# make a dict mapping uuids to data vectors
data_map = dict(zip(uuids, data))

# plot all the data
for timeseries in tags:
  d = data_map[timeseries['uuid']]
  # since we have the tags, we can add some metadata
  label = "%s (%s)" % (timeseries['Metadata/SourceName'],
                       timeseries['Properties/UnitofMeasure'])
コード例 #53
0
from smap.archiver.client import SmapClient
from smap.contrib import dtutil
import numpy as np
import pandas as pd
import datetime
import subprocess

#Link to download the data
c = SmapClient("http://iiitdarchiver.zenatix.com:9105")

#Range of dates to which you want to download the data

start = dtutil.dt2ts(dtutil.strptime_tz("01-10-2017", "%d-%m-%Y"))
end = dtutil.dt2ts(dtutil.strptime_tz("01-10-2017", "%d-%m-%Y"))

# hard-code the UUIDs we want to download
oat = ["eec41258-f057-591e-9759-8cfdeb67b9af"]

# Function to perform the download of the data
data = c.data_uuid(oat, start, end)

t = np.array(data)
df = pd.DataFrame(t)

# creating files after downloading
for i, j in enumerate(t):
    name = str(i) + '.csv'
    with open(name, 'w') as f:
        for time, val in j:
            f.write(
                str(datetime.datetime.fromtimestamp(time / 1000.0)) + ' , ' +