def load_crossection(self, *args):
start = self.start_dt
end = self.start_dt + self.load_chunk_size
if end > self.end_dt: end = self.end_dt
log.msg("loading " + str(self.load_offset) + " " +
str(start) + ' - ' + str(end))
start, end = dtutil.dt2ts(start), \
dtutil.dt2ts(end)
d = threads.deferToThread(self.arclient.data_uuid,
self.load_uids[self.load_offset:
self.load_offset +
self.load_xsec_size],
start, end,
self.cache)
d.addCallback(self.load_data, self.load_offset)
d.addCallback(lambda _: (self._flush(), None))
self.load_offset += self.load_xsec_size
if self.load_offset >= len(self.load_uids):
# pick a new window
self.start_dt += self.load_chunk_size
d.addCallback(self.load_time_chunk)
else:
d.addCallback(self.load_crossection)
def err(e):
print e
d.addErrback(err)
return d
def _update(self, val):
print "update using", val
try:
point, value, quality, time = val
if quality != 'Good':
log.msg("bad quality on point " + point + ": " + quality)
return
# parse the timestamp in the timezone of the server
if self.use_opc_timestamps == 'true':
ts = dtutil.strptime_tz(time,
self.opc_timefmt,
self.opc_timezone)
ts = dtutil.dt2ts(ts)
else:
ts = dtutil.dt2ts(dtutil.now())
path = self.make_path(point)
series = self.get_timeseries(path)
if series:
if series['Properties']['ReadingType'] == 'double':
series._add(ts, float(value))
else:
series._add(ts, int(value))
except:
log.err()
def load_data(self, st, et):
st_utc = dtutil.dt2ts(st)
et_utc = dtutil.dt2ts(et)
ts = int(st_utc / 120) * 120 # round down to nearest 2-min increment
while ts <= et_utc:
self.add('/sensor0', ts, self.counter)
self.counter += 1
ts += 120 # 2-min increments
def load_data(self, st, et):
st_utc = dtutil.dt2ts(st)
et_utc = dtutil.dt2ts(et)
ts = int(st_utc / 120) * 120 # round down to nearest 2-min increment
while ts <= et_utc:
self.add('/sensor0', ts, self.counter)
self.counter += 1
ts += 120 # 2-min increments
def _update(self):
vals = self.opc.read(group="smap-points-group")
for point, value, quality, time in vals:
# parse the timestamp in the timezone of the server
if time is not None:
ts = dtutil.strptime_tz(time, self.opc_timefmt, self.opc_timezone)
ts = dtutil.dt2ts(ts)
else:
ts = dtutil.now(self.opc_timezone)
ts = dtutil.dt2ts(ts)
if self.get_timeseries(self.make_path(point)) and value is not None:
if isinstance(value, bool): value = int(value)
self._add(self.make_path(point), ts, float(value))
def _update(self):
vals = self.opc.read(group="smap-points-group")
for point, value, quality, time in vals:
# parse the timestamp in the timezone of the server
if time is not None:
ts = dtutil.strptime_tz(time, self.opc_timefmt,
self.opc_timezone)
ts = dtutil.dt2ts(ts)
else:
ts = dtutil.now(self.opc_timezone)
ts = dtutil.dt2ts(ts)
if self.get_timeseries(
self.make_path(point)) and value is not None:
if isinstance(value, bool): value = int(value)
self._add(self.make_path(point), ts, float(value))
def test_fill_missing(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
# check that we fill the end correctly
op = grouping.GroupByDatetimeField(self.inputs,
arithmetic.first,
field='hour',
width=1,
skip_missing=False)
now *= 1000
rv = op(
operators.DataChunk((now, now + ((self.hours + 5) * 3600 * 1000)),
True, True, [self.testdata]))
self.assertEqual(len(rv[0]), self.hours + 5)
self.assertEqual(np.sum(np.isnan(rv[0][-5:, 1])), 5)
self.assertEqual(np.sum(np.isnan(rv[0][:-5, 1])), 0)
# and the beginning
op = grouping.GroupByDatetimeField(self.inputs,
arithmetic.first,
field='hour',
width=1,
skip_missing=False)
rv = op(
operators.DataChunk(
(now - (5 * 3600 * 1000), now + ((self.hours) * 3600 * 1000)),
True, True, [self.testdata]))
self.assertEqual(len(rv[0]), self.hours + 5)
self.assertEqual(np.sum(np.isnan(rv[0][:5, 1])), 5)
self.assertEqual(np.sum(np.isnan(rv[0][5:, 1])), 0)
def _update(self):
vals = self.opc.read(group="smap-points-group")
for point, value, quality, time in vals:
# parse the timestamp in the timezone of the server
ts = dtutil.strptime_tz(time, self.opc_timefmt, self.opc_timezone)
ts = dtutil.dt2ts(ts)
self._add(self.make_path(point), ts, value)
def _update(self):
vals = self.opc.read(group="smap-points-group")
for point, value, quality, time in vals:
# parse the timestamp in the timezone of the server
ts = dtutil.strptime_tz(time, self.opc_timefmt, self.opc_timezone)
ts = dtutil.dt2ts(ts)
self._add(self.make_path(point), ts, value)
def read(self):
object_ = {}
print 'read running'
try:
#get the text from the ur
wa = urllib2.urlopen(
'http://transmission.bpa.gov/business/operations/wind/baltwg.txt'
)
data = [
line for line in wa.readlines()[7:] if len(line.split()) > 3
]
#parse most recent data
rawTime = " ".join(data[-1].split()[:2])
currentTime = int(
dtutil.dt2ts(
dtutil.strptime_tz(rawTime, "%m/%d/%Y %H:%M",
'US/Pacific')))
object_["Wind"] = data[-1].split()[3]
object_["Hydro"] = data[-1].split()[4]
object_["Thermal"] = data[-1].split()[5]
object_["Load"] = data[-1].split()[2]
except Exception as e:
logging.exception(type(e))
print e
else:
if currentTime != self.previousTime:
self.w.add(currentTime, int(object_["Wind"]))
self.h.add(currentTime, int(object_["Hydro"]))
self.t.add(currentTime, int(object_["Thermal"]))
self.l.add(currentTime, int(object_["Load"]))
self.previousTime = currentTime
wa.close()
def read(self):
object_ = {}
print 'read running'
try:
#get the text from the ur
wa = urllib2.urlopen('http://transmission.bpa.gov/business/operations/wind/baltwg.txt')
data = [line for line in wa.readlines()[7:] if len(line.split()) > 3]
#parse most recent data
rawTime = " ".join(data[-1].split()[:2])
currentTime = int(dtutil.dt2ts(dtutil.strptime_tz(rawTime,"%m/%d/%Y %H:%M",'US/Pacific')))
object_["Wind"] = data[-1].split()[3]
object_["Hydro"] = data[-1].split()[4]
object_["Thermal"] = data[-1].split()[5]
object_["Load"] = data[-1].split()[2]
except Exception as e:
logging.exception(type(e))
print e
else:
if currentTime != self.previousTime:
self.w.add(currentTime,int(object_["Wind"]))
self.h.add(currentTime,int(object_["Hydro"]))
self.t.add(currentTime,int(object_["Thermal"]))
self.l.add(currentTime,int(object_["Load"]))
self.previousTime = currentTime
wa.close()
def test_inclusive(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
# startshape = self.testdata.shape
# startdata = np.copy(self.testdata)
# op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
# rv = op([self.testdata[:30, :]])
# self.assertEquals(rv[0].shape, (24, 2))
# # check for mutations
# self.assertEquals(self.testdata.shape, startshape)
# self.assertEquals(np.sum(startdata - self.testdata), 0)
op2 = grouping.GroupByDatetimeField(self.inputs,
oputils.NullOperator,
field='day',
inclusive=(True, True),
snap_times=False)
rv = op2([self.testdata[0:30, :]])
self.assertEquals(rv[0].shape, (25, 2))
self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
self.assertEquals(rv[0][24, 0], self.testdata[24, 0])
def test_fill_missing(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
# check that we fill the end correctly
op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
field='hour',
width=1,
skip_empty=False)
now *= 1000
rv = op(operators.DataChunk((now,
now + ((self.hours + 5) * 3600 * 1000)),
True, True,
[self.testdata]))
self.assertEqual(len(rv[0]), self.hours + 5)
self.assertEqual(np.sum(np.isnan(rv[0][-5:, 1])), 5)
self.assertEqual(np.sum(np.isnan(rv[0][:-5, 1])), 0)
# and the beginning
op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
field='hour',
width=1,
skip_empty=False)
rv = op(operators.DataChunk((now - (5 * 3600 * 1000),
now + ((self.hours) * 3600 * 1000)),
True, True,
[self.testdata]))
self.assertEqual(len(rv[0]), self.hours + 5)
self.assertEqual(np.sum(np.isnan(rv[0][:5, 1])), 5)
self.assertEqual(np.sum(np.isnan(rv[0][5:, 1])), 0)
def p_timeref(t):
"""timeref : abstime
| abstime reltime"""
t[0] = t[1]
if len(t) == 2:
ref = t[1]
else:
ref = t[1] + t[2]
t[0] = dtutil.dt2ts(ref) * 1000
def p_timeref(t):
"""timeref : abstime
| abstime reltime"""
t[0] = t[1]
if len(t) == 2:
ref = t[1]
else:
ref = t[1] + t[2]
t[0] = dtutil.dt2ts(ref) * 1000
def test_snap_times(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
field='day',
snap_times=True)
rv = op([self.testdata[10:30]])
self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
def test_offset(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
for i in xrange(0, 24):
rv = op([self.testdata[i:25+i, :]])
self.assertEquals(rv[0].shape, (24 - i, 2))
op.reset()
def setUp(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.testdata = np.ones((self.hours, 2))
for i in xrange(0, self.hours):
self.testdata[i, :] = i
self.testdata[:, 0] *= 3600
self.testdata[:, 0] += now
self.ma = grouping.MaskedDTList(self.testdata[:, 0], dtutil.gettz("America/Los_Angeles"))
self.width = datetime.timedelta(days=1)
def test_slide(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
field='hour',
width=4,
slide=2)
rv = op([self.testdata])
self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[:-2:2, 0]), 0)
self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:-2:2, 1]), 0)
def test_snap_times(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs,
arithmetic.first,
field='day',
snap_times=True)
rv = op([self.testdata[10:30]])
self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
def process(self, doc):
doc = bs(doc)
now = doc.livedata.gatewaytime
now = dtutil.strptime_tz("%s %s %s %s %s %s" % (now.month.contents[0],
now.day.contents[0],
now.year.contents[0],
now.hour.contents[0],
now.minute.contents[0],
now.maxsecond.contents[0]),
"%m %d %y %H %M %S", tzstr=self.timezone)
now = dtutil.dt2ts(now)
self.add('/voltage', now, int(doc.livedata.voltage.total.voltagenow.contents[0]))
self.add('/real_power', now, int(doc.livedata.power.total.powernow.contents[0]))
self.add('/apparent_power', now, int(doc.livedata.power.total.kva.contents[0]))
def test_offset(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs,
oputils.NullOperator,
field='day')
for i in xrange(0, 24):
rv = op([self.testdata[i:25 + i, :]])
self.assertEquals(rv[0].shape, (24 - i, 2))
op.reset()
def setUp(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.testdata = np.ones((self.hours, 2))
for i in xrange(0, self.hours):
self.testdata[i, :] = i
self.testdata[:, 0] *= 3600
self.testdata[:, 0] += now
self.ma = grouping.MaskedDTList(self.testdata[:, 0],
dtutil.gettz("America/Los_Angeles"))
self.width = datetime.timedelta(days=1)
def test_slide(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs,
arithmetic.first,
field='hour',
width=4,
slide=2)
rv = op([self.testdata])
self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[:-2:2, 0]), 0)
self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:-2:2, 1]), 0)
def test_flush(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
field='hour',
width=1)
rv = op(operators.DataChunk((now * 1000,
now * 1000 + (self.hours * 3600 * 1000)),
True, True,
[self.testdata]))
# if we don't properly flush the last hour, we should only get hours - 1 results
self.assertEquals((rv[0][-1, 0] - (now * 1000)) / (3600 * 1000), self.hours - 1)
def _meter_sample(
data,
slop=datetime.timedelta(minutes=30),
bin_equal=_day_bin_equal,
# state parameters
prev=None, # start of meter region
trianglestart=None, # last reset
accum=0, # accumulated energy in window
prev_data=None): # last point we saw
rv = []
if len(data) and (not prev or not trianglestart):
prev = data[0]
prev_data = data[0]
trianglestart = data[0]
accum = 0
start = 1
else:
start = 0
for i, v in enumerate(data[start:]):
if v[1] < trianglestart[1]:
# if we roll over, add in the accumulated sum through now
accum += (prev_data[1] - trianglestart[1])
trianglestart = v
if bin_equal(prev[0], v[0]):
# continue if we're still in the same bin
prev_data = v
continue
elif bin_equal(prev[0], v[0] - slop) and \
not bin_equal(prev[0] - slop, prev[0]):
# otherwise produce a record if the endpoints are close to
accum += prev_data[1] - trianglestart[1]
print "add", prev[0], accum, trianglestart[0]
rv.append((dtutil.dt2ts(prev[0]), accum))
prev_data = v
prev = v
trianglestart = v
accum = 0
return np.array(rv), {
'prev': prev,
'prev_data': prev_data,
'trianglestart': trianglestart,
'accum': accum
}
def OnMultipleItemsChanged(self, sender, args):
for v in args.ArgsArray:
driver = OPC_DRIVERS[v.State]
if v.Exception:
continue
print v.ItemDescriptor.ItemId,
print v.Vtq.Timestamp, v.Vtq.Value
ts = dtutil.strptime_tz(str(v.Vtq.Timestamp),
"%m/%d/%y %H:%M:%S",
tzstr=driver.opc_timezone)
path = driver.make_path(v.ItemDescriptor.ItemId)
try:
driver._add(driver.make_path(v.ItemDescriptor.ItemId),
int(dtutil.dt2ts(ts)), v.Vtq.Value)
except Exception, e:
log.err("Error adding data: " + str(e))
def _meter_sample(data,
slop = datetime.timedelta(minutes=30),
bin_equal=_day_bin_equal,
# state parameters
prev=None, # start of meter region
trianglestart=None, # last reset
accum=0, # accumulated energy in window
prev_data=None): # last point we saw
rv = []
if len(data) and (not prev or not trianglestart):
prev = data[0]
prev_data = data[0]
trianglestart = data[0]
accum = 0
start = 1
else:
start = 0
for i, v in enumerate(data[start:]):
if v[1] < trianglestart[1]:
# if we roll over, add in the accumulated sum through now
accum += (prev_data[1] - trianglestart[1])
trianglestart = v
if bin_equal(prev[0], v[0]):
# continue if we're still in the same bin
prev_data = v
continue
elif bin_equal(prev[0], v[0] - slop) and \
not bin_equal(prev[0] - slop, prev[0]):
# otherwise produce a record if the endpoints are close to
accum += prev_data[1] - trianglestart[1]
print "add", prev[0], accum, trianglestart[0]
rv.append((dtutil.dt2ts(prev[0]), accum))
prev_data = v
prev = v
trianglestart = v
accum = 0
return np.array(rv), {
'prev' : prev,
'prev_data' : prev_data,
'trianglestart' : trianglestart,
'accum' : accum
}
def test_oneatatime(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
for i in xrange(0, 24):
rv = op([self.testdata[i:i+1, :]])
self.assertEquals(rv[0].shape, operators.null.shape)
rv = op([self.testdata[24:25, :]])
self.assertEquals(rv[0].shape, (24, 2))
# make sure we snapped
self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[0, 0]), 0)
# and got back the right data
self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:24, 1]), 0)
def process(self, doc):
doc = bs(doc)
now = doc.livedata.gatewaytime
now = dtutil.strptime_tz(
"%s %s %s %s %s %s" %
(now.month.contents[0], now.day.contents[0], now.year.contents[0],
now.hour.contents[0], now.minute.contents[0],
now.maxsecond.contents[0]),
"%m %d %y %H %M %S",
tzstr=self.timezone)
now = dtutil.dt2ts(now)
self.add('/voltage', now,
int(doc.livedata.voltage.total.voltagenow.contents[0]))
self.add('/real_power', now,
int(doc.livedata.power.total.powernow.contents[0]))
self.add('/apparent_power', now,
int(doc.livedata.power.total.kva.contents[0]))
def OnMultipleItemsChanged(self, sender, args):
for v in args.ArgsArray:
driver = OPC_DRIVERS[v.State]
if v.Exception:
continue
print v.ItemDescriptor.ItemId,
print v.Vtq.Timestamp, v.Vtq.Value
ts = dtutil.strptime_tz(str(v.Vtq.Timestamp),
"%m/%d/%y %H:%M:%S",
tzstr=driver.opc_timezone)
path = driver.make_path(v.ItemDescriptor.ItemId)
try:
driver._add(driver.make_path(v.ItemDescriptor.ItemId),
int(dtutil.dt2ts(ts)),
v.Vtq.Value)
except Exception, e:
log.err("Error adding data: " + str(e))
def test_increment(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
for incr in [2, 4, 6, 8, 12, 24]:
op = grouping.GroupByDatetimeField(self.inputs, arithmetic.first,
field='hour',
width=incr)
rv = op([self.testdata[:25, :]])
# check the shape
self.assertEquals(len(rv[0]), 24 / incr)
for i in xrange(0, 24/incr):
# the timestamps
self.assertEquals(rv[0][i, 0], self.testdata[i * incr, 0])
# and the values
self.assertEquals(rv[0][i, 1], i * incr)
del op
def test_flush(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs,
arithmetic.first,
field='hour',
width=1)
rv = op(
operators.DataChunk(
(now * 1000, now * 1000 + (self.hours * 3600 * 1000)), True,
True, [self.testdata]))
# if we don't properly flush the last hour, we should only get hours - 1 results
self.assertEquals((rv[0][-1, 0] - (now * 1000)) / (3600 * 1000),
self.hours - 1)
def process_one(self, data, tz,
prev=None,
prev_datetimes=None):
times, values = data[:,0], data[:,1]
if (prev is not None):
times = np.append(prev_datetimes, times)
values = np.append(prev, values)
st = self.snapper(dtutil.ts2dt(prev_datetimes[-1] / 1000))
else:
st = self.snapper(dtutil.ts2dt(times[0] / 1000))
st = dtutil.dt2ts(st) * 1000 + self.width
et = int(times[-1])
mesh = np.arange(st, et, self.width)
if (self.max_time_delta):
gaps = self.detect_gaps(times)
remove = np.array([False] * len(mesh))
for gap in gaps:
gt = np.greater(mesh, gap[0])
lt = np.less(mesh, gap[1])
this_gap = np.logical_and(gt, lt)
remove = np.logical_or(remove, this_gap)
remove_inds = np.nonzero(remove)[0]
mesh = np.delete(mesh, remove_inds)
if (self.method == 'linear'):
outvals = np.interp(mesh, times, values)
prev = np.array([values[-1]])
prev_datetimes = np.array([times[-1]])
elif (self.method == 'spline'):
s = UnivariateSpline(times, values, s=0)
outvals = s(mesh)
# 10 points = empirical
prev = np.array(values[-10:])
prev_datetimes = np.array(times[-10:])
output = np.vstack((mesh,outvals)).T
state = { 'prev': prev,
'prev_datetimes': prev_datetimes,
}
return output, state
def process_one(self, data, tz, prev=None, prev_datetimes=None):
times, values = data[:, 0], data[:, 1]
if (prev is not None):
times = np.append(prev_datetimes, times)
values = np.append(prev, values)
st = self.snapper(dtutil.ts2dt(prev_datetimes[-1] / 1000))
else:
st = self.snapper(dtutil.ts2dt(times[0] / 1000))
st = dtutil.dt2ts(st) * 1000 + self.width
et = int(times[-1])
mesh = np.arange(st, et, self.width)
if (self.max_time_delta):
gaps = self.detect_gaps(times)
remove = np.array([False] * len(mesh))
for gap in gaps:
gt = np.greater(mesh, gap[0])
lt = np.less(mesh, gap[1])
this_gap = np.logical_and(gt, lt)
remove = np.logical_or(remove, this_gap)
remove_inds = np.nonzero(remove)[0]
mesh = np.delete(mesh, remove_inds)
if (self.method == 'linear'):
outvals = np.interp(mesh, times, values)
prev = np.array([values[-1]])
prev_datetimes = np.array([times[-1]])
elif (self.method == 'spline'):
s = UnivariateSpline(times, values, s=0)
outvals = s(mesh)
# 10 points = empirical
prev = np.array(values[-10:])
prev_datetimes = np.array(times[-10:])
output = np.vstack((mesh, outvals)).T
state = {
'prev': prev,
'prev_datetimes': prev_datetimes,
}
return output, state
def test_increment(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
for incr in [2, 4, 6, 8, 12, 24]:
op = grouping.GroupByDatetimeField(self.inputs,
arithmetic.first,
field='hour',
width=incr)
rv = op([self.testdata[:25, :]])
# check the shape
self.assertEquals(len(rv[0]), 24 / incr)
for i in xrange(0, 24 / incr):
# the timestamps
self.assertEquals(rv[0][i, 0], self.testdata[i * incr, 0])
# and the values
self.assertEquals(rv[0][i, 1], i * incr)
del op
def test_oneatatime(self):
now = dtutil.strptime_tz("1 1 2000 0",
"%m %d %Y %H",
tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
op = grouping.GroupByDatetimeField(self.inputs,
oputils.NullOperator,
field='day')
for i in xrange(0, 24):
rv = op([self.testdata[i:i + 1, :]])
self.assertEquals(rv[0].shape, operators.null.shape)
rv = op([self.testdata[24:25, :]])
self.assertEquals(rv[0].shape, (24, 2))
# make sure we snapped
self.assertEquals(np.sum(rv[0][:, 0] - self.testdata[0, 0]), 0)
# and got back the right data
self.assertEquals(np.sum(rv[0][:, 1] - self.testdata[:24, 1]), 0)
def read(self):
for retry_time in [0, 30, 5 * 60]:
time.sleep(retry_time)
try:
print "Reading"
data = urllib2.urlopen(self.url, timeout=30).read()
times = {}
b = BeautifulSoup.BeautifulSoup(data)
# Parse time blocks
data = b.find('data')
for time_block in data.findAll('time-layout'):
key = time_block.find('layout-key').contents[0]
time_list = []
for time_tag in time_block.findAll('start-valid-time'):
#dt = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S%z')
dt = dateutil.parser.parse(time_tag.string)
time_list.append(dtutil.dt2ts(dt))
times[key] = time_list
# For each value block find referenced time block and add readings
for data_block in data.find('parameters').findAll(
recursive=False):
key = data_block['time-layout']
# Find the element being returned
for (name, e) in self.element_map.items():
if e.tag_name == data_block.name and e.type_name == data_block[
'type']:
# Element found
value = []
for v in data_block.findAll('value'):
value.append(float(v.string))
for t, v in zip(times[key], value):
self.add('/' + name, int(t), v)
break
return
except Exception, e:
print e
def test_inclusive(self):
now = dtutil.strptime_tz("1 1 2000 0", "%m %d %Y %H", tzstr="America/Los_Angeles")
now = dtutil.dt2ts(now)
self.setUp(now)
# startshape = self.testdata.shape
# startdata = np.copy(self.testdata)
# op = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator, field='day')
# rv = op([self.testdata[:30, :]])
# self.assertEquals(rv[0].shape, (24, 2))
# # check for mutations
# self.assertEquals(self.testdata.shape, startshape)
# self.assertEquals(np.sum(startdata - self.testdata), 0)
op2 = grouping.GroupByDatetimeField(self.inputs, oputils.NullOperator,
field='day', inclusive=(True, True),
snap_times=False)
rv = op2([self.testdata[0:30, :]])
self.assertEquals(rv[0].shape, (25, 2))
self.assertEquals(rv[0][0, 0], self.testdata[0, 0])
self.assertEquals(rv[0][24, 0], self.testdata[24, 0])
def read(self):
for retry_time in [0, 30, 5*60]:
time.sleep(retry_time)
try:
print "Reading"
data = urllib2.urlopen(self.url, timeout = 30).read()
times = {}
b = BeautifulSoup.BeautifulSoup(data)
# Parse time blocks
data = b.find('data')
for time_block in data.findAll('time-layout'):
key = time_block.find('layout-key').contents[0]
time_list = []
for time_tag in time_block.findAll('start-valid-time'):
#dt = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S%z')
dt = dateutil.parser.parse(time_tag.string)
time_list.append(dtutil.dt2ts(dt))
times[key] = time_list
# For each value block find referenced time block and add readings
for data_block in data.find('parameters').findAll(recursive=False):
key = data_block['time-layout']
# Find the element being returned
for (name, e) in self.element_map.items():
if e.tag_name == data_block.name and e.type_name == data_block['type']:
# Element found
value = []
for v in data_block.findAll('value'):
value.append(float(v.string))
for t,v in zip(times[key], value):
self.add('/'+name, int(t), v)
break
return
except Exception, e:
print e
def process(self):
readcnt = 0
data = []
if self.reader == None:
return data
try:
for r in self.reader:
ts = dtutil.strptime_tz(r[0], TIMEFMT, tzstr='UTC')
if ts > self.push_hist:
self.push_hist = ts
ts = dtutil.dt2ts(ts)
data.append((ts, zip(self.field_map, r)))
readcnt += 1
if readcnt > 100:
return data
except Exception, e:
self.fp.close()
self.reader = None
raise e
def process(self, body):
reader = csv.reader(StringIO.StringIO(body), dialect='excel-tab')
header = reader.next()
if len(header) == 0:
print "Warning: no data from", self.url
raise core.SmapException("no data!")
try:
self.field_map, self.map = make_field_idxs(self.meter_type, header,
location=self.location)
except:
traceback.print_exc()
if not self.added:
self.added = True
for channel in self.map['sensors'] + self.map['meters']:
try:
self.add_timeseries('/%s/%s' % channel[2:4], channel[4], data_type='double')
self.set_metadata('/%s/%s' % channel[2:4], {
'Extra/ChannelName' : re.sub('\(.*\)', '', channel[0]).strip(),
})
except:
traceback.print_exc()
# add all the values
for r in reader:
ts = dtutil.strptime_tz(r[0], TIMEFMT, tzstr='UTC')
if ts > self.push_hist:
self.push_hist = ts
ts = dtutil.dt2ts(ts)
for descr, val in zip(self.field_map, r):
if descr == None: continue
try:
self._add('/' + '/'.join(descr), ts, float(val))
except ValueError:
pass
bldg = bldg_dict[int(num)]
# start = raw_input("start time (\"%m-%d-%Y %H:%M\" or "-d" for default): ")
# end = raw_input("end time (\"%m-%d-%Y %H:%M\") or "-d" for default: ")
start = "10-21-2013 00:00"
end = "10-27-2013 23:59"
# get the outside air temperature during the period specified
# get_temp(start, end)
if start == '-d':
print "computing the IMFs in %s from 7 days backwards till now..." %bldg
else:
print "computing the IMFs in %s from %s to %s..." %(bldg, start, end)
if end == '-d':
end = int(time.time()*1000)
else:
end = dtutil.dt2ts(dtutil.strptime_tz("%s" %end, "%m-%d-%Y %H:%M")) * 1000
if start == '-d':
start = end - 7*24*60*60*1000
else:
start = dtutil.dt2ts(dtutil.strptime_tz("%s" %start, "%m-%d-%Y %H:%M")) * 1000
applySum = "apply nansum(axis=1) < paste < window(first, field='minute', width=15) < units to data in (%f, %f) limit -1 streamlimit 10000 \
where (Metadata/Extra/System = 'total' or Metadata/Extra/System = 'electric') \
and (Properties/UnitofMeasure = 'kW' or Properties/UnitofMeasure = 'Watts' or Properties/UnitofMeasure = 'W') \
and Metadata/Location/Building like '%s%%' and not Metadata/Extra/Operator like 'sum%%' \
and not Path like '%%demand' and not Path like '/Cory_Hall/Electric_5A7/ABC/real_power' and not Path like '/Cory_Hall/Electric_5B7/ABC/real_power'" \
%(start, end, bldg)
result = c.query(applySum) #the result is a list
reading = result[0]['Readings']
# output readings to file
def process_one(self, data, op, tz,
prev=None,
prev_datetimes=None,
first=False, last=False,
region=(None, None)):
# print "PRCESSING"
tic = time.time()
if prev == None:
prev = np.copy(data)
prev_datetimes = MaskedDTList(prev[:, 0] / 1000, tz)
else:
prev = np.vstack((prev, data))
prev_datetimes.extend(data[:, 0] / 1000)
assert len(prev_datetimes) == len(prev)
output = [null] * len(op.outputs)
# output = [null] * len(util.flatten(map(operator.attrgetter("outputs"), ops)))
# print output
if len(prev_datetimes) == 0:
return output, {
'prev': prev,
'prev_datetimes': prev_datetimes,
}
# we might want to produce readings before the first data point
if first and region[0]:
bin_start = self.snapper(dtutil.ts2dt(region[0] / 1000))
else:
bin_start = self.snapper(prev_datetimes[0])
truncate_to = 0
while True:
if last:
if not region[1] and truncate_to == len(prev_datetimes):
break
if region[1] and region[1] <= dtutil.dt2ts(bin_start) * 1000:
break
bin_end = bin_start + self.bin_slide
# perform a binary search to find the next window boundary
bin_start_idx = bisect.bisect_left(prev_datetimes, bin_start)
bin_end_idx = bisect.bisect_left(prev_datetimes, bin_end)
truncate_to = bin_start_idx
# ignore bins which aren't full
if bin_end_idx == len(prev_datetimes) and not last:
break
# skip empty bins
if bin_start_idx == bin_end_idx:
# maybe we were supposed to produce output even if
# there's no data in the bin
if not self.skip_empty:
t = dtutil.dt2ts(bin_start) * 1000
output = extend(output,
[np.array([[t, np.nan]])])
bin_start += self.bin_slide
continue
if (bin_end_idx < len(prev_datetimes) and
self.comparator(bin_start, prev_datetimes[bin_end_idx])):
take_end = bin_end_idx + 1
else:
take_end = bin_end_idx
opdata = op([prev[bin_start_idx:take_end, :]])
# snap the times to the beginning of the
# window, if we were asked to. do this here
# so we can avoid passing datetimes around,
# and deal with the common case where this is
# what ya want.
if self.snap_times:
t = dtutil.dt2ts(bin_start)
for j in xrange(0, len(opdata)):
opdata[j][:, 0] = t * 1000
output = extend(output, opdata)
bin_start += self.bin_slide
toc = time.time()
# print("dt processing took %0.05f: %i/%i converted" % \
# (toc-tic,
# prev_datetimes.conversions,
# len(prev_datetimes)))
prev_datetimes.truncate(truncate_to)
prev = prev[truncate_to:]
return output, {
'prev': prev,
'prev_datetimes': prev_datetimes,
}
from smap.archiver.client import SmapClient
from smap.contrib import dtutil
from matplotlib import pyplot
from matplotlib import dates
import os
# make a client
c = SmapClient("http://new.openbms.org/backend")
# start and end values are Unix timestamps
t_start = "6-12-2013 8:00"
t_end = "6-19-2013 8:00"
start = 1000*dtutil.dt2ts(dtutil.strptime_tz(t_start, "%m-%d-%Y %H:%M"))
end = 1000*dtutil.dt2ts(dtutil.strptime_tz(t_end, "%m-%d-%Y %H:%M"))
stnc = "select distinct Metadata/Location/RoomNumber where Metadata/SourceName='KETI Motes'"
roomlist = c.query(stnc) #the result is a list
#roomlist = roomlist[16:]
#roomlist = ['621A','621B','621C','621D','621E']
for room in roomlist:
print "==========Fetching streams in Room %s=========="%room
stnc = "select Path where Metadata/Location/RoomNumber='%s' and not Path ~ '.*pir.*'" %room
streams = c.query(stnc)
if len(streams)>0:
# print "----%d streams in Room %s----"%(len(streams), room)
for s in streams:
# fetch the metadata of path wanted
tags = c.tags("Path='%s'"%s['Path'])
def get_data(self,uuid,start,end,limit=-1):
# print start
# print end
startTime = dtutil.dt2ts(dtutil.strptime_tz(start, "%m/%d/%Y %H:%M:%S %p" ))
endTime = dtutil.dt2ts(dtutil.strptime_tz(end, "%m/%d/%Y %H:%M:%S %p"))
return self.c.data_uuid(uuid, startTime, endTime,True,limit)
def parse_time(self, ts, val):
if self.timefmt == None:
return int(val)
else:
return dtutil.dt2ts(
dtutil.strptime_tz(val, self.timefmt, self.timezone))
search = keywordSearch(lines, "sdh room temp", 2000) add = keywordSearch(lines, "sdh rm temp", 1000) for i in water: res.remove(i) for i in stp: res.remove(i) for r in res: search_path.append(r['Path']) for i in add: search.append(i) for i in search: search_res.append(i['Path']) train_start = "6-12-2013 8:00" train_end = "6-12-2013 8:10" start1 = 1000*dtutil.dt2ts(dtutil.strptime_tz(train_start, "%m-%d-%Y %H:%M")) end1 = 1000*dtutil.dt2ts(dtutil.strptime_tz(train_end, "%m-%d-%Y %H:%M")) test_start = "6-13-2013 10:10" test_end = "6-13-2013 10:20" start2 = 1000*dtutil.dt2ts(dtutil.strptime_tz(test_start, "%m-%d-%Y %H:%M")) end2 = 1000*dtutil.dt2ts(dtutil.strptime_tz(test_end, "%m-%d-%Y %H:%M")) anomaly_path1 = [] anomaly_path2 = [] train_data = [] test_data = [] search_train = [] # anomaly_train = [] search_test = [] # anomaly_test = []
def get_readings(self, market, start_date, stop_date):
readings = {
'total_price': [],
'loss': [],
'energy': [],
'congestion': []
}
print "get_readings", market
if market == 'DAM':
q = 'PRC_LMP'
m = 'DAM'
elif market == 'HASP':
q = 'PRC_HASP_LMP'
m = 'HASP'
elif market == 'RTM':
q = 'PRC_INTVL_LMP'
m = 'RTM'
else:
raise Exception("Invalid market: " + market)
url = 'http://oasis.caiso.com/mrtu-oasis/SingleZip?'
url += 'queryname=' + q
url += '&startdate=' + dtutil.strftime_tz(start_date, '%Y%m%d',
'US/Pacific')
url += '&enddate=' + dtutil.strftime_tz(stop_date, '%Y%m%d',
'US/Pacific')
url += '&market_run_id=' + m
url += '&node=' + self.location
logging.info("Get url %s" % url)
h = None
for d in [5, 20, 60]:
try:
h = urllib2.urlopen(url, timeout=50)
break
except urllib2.URLError:
logging.warn("urlopen failed.")
time.sleep(d)
if h == None:
raise Exception("Failed to open url: %s" % url)
z = zipfile.ZipFile(StringIO.StringIO(h.read()))
xml = z.read(z.namelist()[0])
b = BeautifulSoup.BeautifulSoup(xml)
sec_per_int = int(b.find('m:sec_per_interval').contents[0])
rows = b.findAll('m:report_data')
for d in rows:
res = d.find('m:resource_name').contents[0]
item = d.find('m:data_item').contents[0]
day = d.find('m:opr_date').contents[0]
inter = int(d.find('m:interval_num').contents[0])
val = float(d.find('m:value').contents[0])
secs = (inter - 1) * sec_per_int
dt = dtutil.strptime_tz(
day, '%Y-%m-%d',
'US/Pacific') + datetime.timedelta(seconds=secs)
timestamp = dtutil.dt2ts(dt)
key = None
if item == 'LMP_PRC':
key = 'total_price'
elif item == 'LMP_LOSS_PRC':
key = 'loss'
elif item == 'LMP_ENE_PRC':
key = 'energy'
elif item == 'LMP_CONG_PRC':
key = 'congestion'
else:
continue
readings[key].append((timestamp, val))
num_readings = len(readings[readings.keys()[0]])
for k in readings.keys():
if len(readings[k]) != num_readings:
raise Exception('Missing readings')
readings[k] = sorted(readings[k], key=lambda (t, v): t)
return readings
def get_readings(self, market, start_date, stop_date):
readings = {'total_price': [], 'loss': [], 'energy': [], 'congestion': []}
print "get_readings", market
if market == 'DAM':
q = 'PRC_LMP'
m = 'DAM'
elif market == 'HASP':
q = 'PRC_HASP_LMP'
m = 'HASP'
elif market == 'RTM':
q = 'PRC_INTVL_LMP'
m = 'RTM'
else:
raise Exception("Invalid market: " + market)
url = 'http://oasis.caiso.com/mrtu-oasis/SingleZip?'
url += 'queryname=' + q
url += '&startdate=' + dtutil.strftime_tz(start_date, '%Y%m%d', 'US/Pacific')
url += '&enddate=' + dtutil.strftime_tz(stop_date, '%Y%m%d', 'US/Pacific')
url += '&market_run_id=' + m
url += '&node=' + self.location
logging.info("Get url %s" % url)
h = None
for d in [5, 20, 60]:
try:
h = urllib2.urlopen(url, timeout=50)
break
except urllib2.URLError:
logging.warn("urlopen failed.")
time.sleep(d)
if h == None:
raise Exception("Failed to open url: %s" % url)
z = zipfile.ZipFile(StringIO.StringIO(h.read()))
xml = z.read(z.namelist()[0])
b = BeautifulSoup.BeautifulSoup(xml)
sec_per_int = int( b.find('m:sec_per_interval').contents[0] )
rows = b.findAll('m:report_data')
for d in rows:
res = d.find('m:resource_name').contents[0]
item = d.find('m:data_item').contents[0]
day = d.find('m:opr_date').contents[0]
inter = int( d.find('m:interval_num').contents[0] )
val = float( d.find('m:value').contents[0] )
secs = (inter - 1) * sec_per_int
dt = dtutil.strptime_tz(day, '%Y-%m-%d', 'US/Pacific') + datetime.timedelta(seconds=secs)
timestamp = dtutil.dt2ts(dt)
key = None
if item == 'LMP_PRC':
key = 'total_price'
elif item == 'LMP_LOSS_PRC':
key = 'loss'
elif item == 'LMP_ENE_PRC':
key = 'energy'
elif item == 'LMP_CONG_PRC':
key = 'congestion'
else:
continue
readings[key].append((timestamp, val))
num_readings = len(readings[readings.keys()[0]])
for k in readings.keys():
if len(readings[k]) != num_readings:
raise Exception('Missing readings')
readings[k] = sorted(readings[k], key=lambda (t, v): t)
return readings
def process_one(self,
data,
op,
tz,
prev=None,
prev_datetimes=None,
first=False,
last=False,
region=(None, None)):
# print "PRCESSING"
tic = time.time()
if prev == None:
prev = np.copy(data)
prev_datetimes = MaskedDTList(prev[:, 0] / 1000, tz)
else:
prev = np.vstack((prev, data))
prev_datetimes.extend(data[:, 0] / 1000)
assert len(prev_datetimes) == len(prev)
output = [null] * len(op.outputs)
if len(prev_datetimes) == 0:
return output, {
'prev': prev,
'prev_datetimes': prev_datetimes,
}
# we might want to produce readings before the first data point
if first and region[0]:
bin_start = self.snapper(dtutil.ts2dt(region[0] / 1000))
else:
bin_start = self.snapper(prev_datetimes[0])
truncate_to = 0
while True:
if last:
if not region[1] and truncate_to == len(prev_datetimes):
break
if region[1] and region[1] <= dtutil.dt2ts(bin_start) * 1000:
break
bin_end = bin_start + self.bin_slide
# perform a binary search to find the next window boundary
bin_start_idx = bisect.bisect_left(prev_datetimes, bin_start)
bin_end_idx = bisect.bisect_left(prev_datetimes, bin_end)
truncate_to = bin_start_idx
# ignore bins which aren't full
if (bin_end_idx == len(prev_datetimes) and not last
and not (region[1]
and dtutil.dt2ts(bin_end) * 1000 <= region[1])):
break
# skip empty bins
if bin_start_idx == bin_end_idx:
# maybe we were supposed to produce output even if
# there's no data in the bin
if not self.skip_empty:
t = dtutil.dt2ts(bin_start) * 1000
output = extend(output, [np.array([[t, np.nan]])])
bin_start += self.bin_slide
continue
if (bin_end_idx < len(prev_datetimes) and self.comparator(
bin_start, prev_datetimes[bin_end_idx])):
take_end = bin_end_idx + 1
else:
take_end = bin_end_idx
opdata = op([prev[bin_start_idx:take_end, :]])
# snap the times to the beginning of the
# window, if we were asked to. do this here
# so we can avoid passing datetimes around,
# and deal with the common case where this is
# what ya want.
if self.snap_times:
t = dtutil.dt2ts(bin_start)
for j in xrange(0, len(opdata)):
opdata[j][:, 0] = t * 1000
output = extend(output, opdata)
bin_start += self.bin_slide
toc = time.time()
# print("dt processing took %0.05f: %i/%i converted" % \
# (toc-tic,
# prev_datetimes.conversions,
# len(prev_datetimes)))
prev_datetimes.truncate(truncate_to)
prev = prev[truncate_to:]
return output, {
'prev': prev,
'prev_datetimes': prev_datetimes,
}
locating the streams using a metadata query.
@author Stephen Dawson-Haggerty <*****@*****.**>
"""
from smap.archiver.client import SmapClient
from smap.contrib import dtutil
from matplotlib import pyplot
from matplotlib import dates
# make a client
c = SmapClient("http://www.openbms.org/backend")
# start and end values are Unix timestamps
start = dtutil.dt2ts(dtutil.strptime_tz("1-1-2013", "%m-%d-%Y"))
end = dtutil.dt2ts(dtutil.strptime_tz("1-2-2013", "%m-%d-%Y"))
# download the data and metadata
tags = c.tags("Metadata/Extra/Type = 'oat'")
uuids, data = c.data("Metadata/Extra/Type = 'oat'", start, end)
# make a dict mapping uuids to data vectors
data_map = dict(zip(uuids, data))
# plot all the data
for timeseries in tags:
d = data_map[timeseries['uuid']]
# since we have the tags, we can add some metadata
label = "%s (%s)" % (timeseries['Metadata/SourceName'],
timeseries['Properties/UnitofMeasure'])
from smap.archiver.client import SmapClient
from smap.contrib import dtutil
import numpy as np
import pandas as pd
import datetime
import subprocess
#Link to download the data
c = SmapClient("http://iiitdarchiver.zenatix.com:9105")
#Range of dates to which you want to download the data
start = dtutil.dt2ts(dtutil.strptime_tz("01-10-2017", "%d-%m-%Y"))
end = dtutil.dt2ts(dtutil.strptime_tz("01-10-2017", "%d-%m-%Y"))
# hard-code the UUIDs we want to download
oat = ["eec41258-f057-591e-9759-8cfdeb67b9af"]
# Function to perform the download of the data
data = c.data_uuid(oat, start, end)
t = np.array(data)
df = pd.DataFrame(t)
# creating files after downloading
for i, j in enumerate(t):
name = str(i) + '.csv'
with open(name, 'w') as f:
for time, val in j:
f.write(
str(datetime.datetime.fromtimestamp(time / 1000.0)) + ' , ' +