def truncate(self, before=None, after=None, copy=True):
"""Function truncate a sorted DataFrame / Series before and/or after
some particular dates.
Parameters
----------
before : date
Truncate before date
after : date
Truncate after date
Returns
-------
truncated : type of caller
"""
before = datetools.to_datetime(before)
after = datetools.to_datetime(after)
if before is not None and after is not None:
assert (before <= after)
left, right = self.index.slice_locs(before, after)
result = self[left:right]
if isinstance(self.index, MultiIndex):
result.index = self.index.truncate(before, after)
if copy:
result = result.copy()
return result
def _current_time_changed(self, old_time, new_time):
num_of_days = int((datetools.to_datetime(new_time) - self._data_times.min()).days)
if self.Relative_Start_Day != num_of_days:
self.Relative_Start_Day = num_of_days
elif old_time != new_time:
self.current_time = datetools.to_datetime(self.current_time)
self.update()
def truncate(self, before=None, after=None, copy=True):
"""Function truncate a sorted DataFrame / Series before and/or after
some particular dates.
Parameters
----------
before : date
Truncate before date
after : date
Truncate after date
Returns
-------
truncated : type of caller
"""
before = datetools.to_datetime(before)
after = datetools.to_datetime(after)
if before is not None and after is not None:
assert(before <= after)
result = self.ix[before:after]
if isinstance(self.index, MultiIndex):
result.index = self.index.truncate(before, after)
if copy:
result = result.copy()
return result
def __init__(self, start=None, end=None, nPeriods=None,
offset=datetools.BDay(), timeRule=None):
if timeRule is not None:
offset = datetools.getOffset(timeRule)
if timeRule is None:
if offset in datetools._offsetNames:
timeRule = datetools._offsetNames[offset]
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
if start and not offset.onOffset(start):
start = start + offset.__class__(n=1, **offset.kwds)
if end and not offset.onOffset(end):
end = end - offset.__class__(n=1, **offset.kwds)
if nPeriods == None and end < start:
end = None
nPeriods = 0
if end is None:
end = start + (nPeriods - 1) * offset
if start is None:
start = end - (nPeriods - 1) * offset
self.offset = offset
self.timeRule = timeRule
self.start = start
self.end = end
self.nPeriods = nPeriods
def __new__(cls,
start=None,
end=None,
periods=None,
offset=datetools.bday,
timeRule=None,
**kwds):
# Allow us to circumvent hitting the cache
index = kwds.get('index')
if index is None:
if timeRule is not None:
offset = datetools.getOffset(timeRule)
if timeRule is None:
if offset in datetools._offsetNames:
timeRule = datetools._offsetNames[offset]
# Cachable
if not start:
start = kwds.get('begin')
if not end:
end = kwds.get('end')
if not periods:
periods = kwds.get('nPeriods')
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
# inside cache range
fromInside = start is not None and start > CACHE_START
toInside = end is not None and end < CACHE_END
useCache = fromInside and toInside
if (useCache and offset.isAnchored()
and not isinstance(offset, datetools.Tick)):
index = cls.getCachedRange(start,
end,
periods=periods,
offset=offset,
timeRule=timeRule)
else:
xdr = XDateRange(start=start,
end=end,
nPeriods=periods,
offset=offset,
timeRule=timeRule)
index = np.array(list(xdr), dtype=object, copy=False)
index = index.view(cls)
index.offset = offset
else:
index = index.view(cls)
return index
def _sanitize_dates(self, start, end):
start = to_datetime(start)
end = to_datetime(end)
if start is None:
start = datetime.datetime.today() - datetime.timedelta(365)
if end is None:
end = datetime.datetime.today()
return start, end
def _sanitize_dates(start, end):
start = to_datetime(start)
end = to_datetime(end)
if start is None:
start = datetime(1900, 1, 1)
if end is None:
end = datetime.today()
return start, end
def _current_time_changed(self, old_time, new_time):
num_of_days = int(
(datetools.to_datetime(new_time) - self._data_times.min()).days)
if self.Relative_Start_Day != num_of_days:
self.Relative_Start_Day = num_of_days
elif old_time != new_time:
self.current_time = datetools.to_datetime(self.current_time)
self.update()
def _sanitize_dates(self, start, end):
start = to_datetime(start)
end = to_datetime(end)
if start is None:
start = datetime.datetime.today() - datetime.timedelta(365)
if end is None:
end = datetime.datetime.today()
return start, end
def _sanitize_dates(start_date, end_date):
from pandas.core.datetools import to_datetime
start_date = to_datetime(start_date)
end_date = to_datetime(end_date)
if start_date is None:
start_date = DATETIME_START_DEFAULT
if end_date is None:
end_date = datetime.datetime.today()
return start_date, end_date
def _sanitize_dates(start, end):
from pandas.core.datetools import to_datetime
start = to_datetime(start)
end = to_datetime(end)
if start is None:
start = dt.datetime(2010, 1, 1)
if end is None:
end = dt.datetime.today()
return start, end
def _sanitize_dates(start, end):
from pandas.core.datetools import to_datetime
start = to_datetime(start)
end = to_datetime(end)
if start is None:
start = dt.datetime.today() - dt.timedelta(365)
if end is None:
end = dt.datetime.today()
return start, end
def _start_day_changed ( self, new_time ):
if isinstance(new_time, int):
self.current_time = datetools.to_datetime(self._data_times.min()+datetools.Day(new_time))
elif isinstance(new_time, str):
self.current_time = datetools.to_datetime(new_time)
elif isinstance(new_time, datetime.date):
self.current_time = datetools.to_datetime(new_time)
else:
print "Unsupported start day ", new_time
return
def test_to_datetime1():
actual = to_datetime(datetime(2008, 1, 15))
assert actual == datetime(2008, 1, 15)
actual = to_datetime('20080115')
assert actual == datetime(2008, 1, 15)
# unparseable
s = 'Month 1, 1999'
assert to_datetime(s) == s
def test_to_datetime1():
actual = to_datetime(datetime(2008, 1, 15))
assert actual == datetime(2008, 1, 15)
actual = to_datetime('20080115')
assert actual == datetime(2008, 1, 15)
# unparseable
s = 'Month 1, 1999'
assert to_datetime(s) == s
def test_to_datetime1():
actual = to_datetime(datetime(2008, 1, 15))
assert actual == datetime(2008, 1, 15)
actual = to_datetime("20080115")
assert actual == datetime(2008, 1, 15)
# unparseable
s = "Month 1, 1999"
assert to_datetime(s) == s
def _start_day_changed(self, new_time):
if isinstance(new_time, int):
self.current_time = datetools.to_datetime(self._data_times.min() +
datetools.Day(new_time))
elif isinstance(new_time, str):
self.current_time = datetools.to_datetime(new_time)
elif isinstance(new_time, datetime.date):
self.current_time = datetools.to_datetime(new_time)
else:
print "Unsupported start day ", new_time
return
def __new__(cls, start=None, end=None, periods=None,
offset=datetools.bday, time_rule=None,
tzinfo=None, name=None, **kwds):
time_rule = kwds.get('timeRule', time_rule)
if time_rule is not None:
offset = datetools.getOffset(time_rule)
if time_rule is None:
if offset in datetools._offsetNames:
time_rule = datetools._offsetNames[offset]
# Cachable
if not start:
start = kwds.get('begin')
if not periods:
periods = kwds.get('nPeriods')
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
if start is not None and not isinstance(start, datetime):
raise ValueError('Failed to convert %s to datetime' % start)
if end is not None and not isinstance(end, datetime):
raise ValueError('Failed to convert %s to datetime' % end)
# inside cache range. Handle UTC case
useCache = _will_use_cache(offset)
start, end, tzinfo = _figure_out_timezone(start, end, tzinfo)
useCache = useCache and _naive_in_cache_range(start, end)
if useCache:
index = cls._cached_range(start, end, periods=periods,
offset=offset, time_rule=time_rule,
name=name)
if tzinfo is None:
return index
else:
xdr = generate_range(start=start, end=end, periods=periods,
offset=offset, time_rule=time_rule)
index = list(xdr)
if tzinfo is not None:
index = [d.replace(tzinfo=tzinfo) for d in index]
index = np.array(index, dtype=object, copy=False)
index = index.view(cls)
index.name = name
index.offset = offset
index.tzinfo = tzinfo
return index
def _sanitize_dates(start_date, end_date):
"""
Sanitize dates (set to default)
start_date - default: DATETIME_START_DEFAULT
end_date - default: today
"""
from pandas.core.datetools import to_datetime
start_date = to_datetime(start_date)
end_date = to_datetime(end_date)
if start_date is None:
start_date = DATETIME_START_DEFAULT
if end_date is None:
end_date = datetime.datetime.today()
return start_date, end_date
def _sanitize_dates(start_date, end_date):
"""
Sanitize dates (set to default)
start_date - default: DATETIME_START_DEFAULT
end_date - default: today
"""
from pandas.core.datetools import to_datetime
start_date = to_datetime(start_date)
end_date = to_datetime(end_date)
if start_date is None:
start_date = DATETIME_START_DEFAULT
if end_date is None:
end_date = datetime.datetime.today()
return start_date, end_date
def __new__(cls, start=None, end=None, periods=None,
offset=datetools.bday, timeRule=None, **kwds):
# Allow us to circumvent hitting the cache
index = kwds.get('index')
if index is None:
if timeRule is not None:
offset = datetools.getOffset(timeRule)
if timeRule is None:
if offset in datetools._offsetNames:
timeRule = datetools._offsetNames[offset]
# Cachable
if not start:
start = kwds.get('begin')
if not end:
end = kwds.get('end')
if not periods:
periods = kwds.get('nPeriods')
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
# inside cache range
fromInside = start is not None and start > CACHE_START
toInside = end is not None and end < CACHE_END
useCache = fromInside and toInside
if (useCache and offset.isAnchored() and
not isinstance(offset, datetools.Tick)):
index = cls.getCachedRange(start, end, periods=periods,
offset=offset, timeRule=timeRule)
else:
xdr = XDateRange(start=start, end=end,
nPeriods=periods, offset=offset,
timeRule=timeRule)
index = np.array(list(xdr), dtype=object, copy=False)
index = index.view(cls)
index.offset = offset
else:
index = index.view(cls)
return index
def test_parse_dates_column_list(self):
from pandas.core.datetools import to_datetime
data = '''date;destination;ventilationcode;unitcode;units;aux_date
01/01/2010;P;P;50;1;12/1/2011
01/01/2010;P;R;50;1;13/1/2011
15/01/2010;P;P;50;1;14/1/2011
01/05/2010;P;P;50;1;15/1/2011'''
expected = read_csv(StringIO(data), sep=";", index_col=range(4))
lev = expected.index.levels[0]
expected.index.levels[0] = lev.to_datetime(dayfirst=True)
expected['aux_date'] = to_datetime(expected['aux_date'], dayfirst=True)
expected['aux_date'] = map(Timestamp, expected['aux_date'])
self.assert_(isinstance(expected['aux_date'][0], datetime))
df = read_csv(StringIO(data),
sep=";",
index_col=range(4),
parse_dates=[0, 5],
dayfirst=True)
assert_frame_equal(df, expected)
df = read_csv(StringIO(data),
sep=";",
index_col=range(4),
parse_dates=['date', 'aux_date'],
dayfirst=True)
assert_frame_equal(df, expected)
def test_parse_dates_column_list(self):
from pandas.core.datetools import to_datetime
data = '''date;destination;ventilationcode;unitcode;units;aux_date
01/01/2010;P;P;50;1;12/1/2011
01/01/2010;P;R;50;1;13/1/2011
15/01/2010;P;P;50;1;14/1/2011
01/05/2010;P;P;50;1;15/1/2011'''
expected = read_csv(StringIO(data), sep=";", index_col=range(4))
lev = expected.index.levels[0]
expected.index.levels[0] = lev.to_datetime(dayfirst=True)
expected['aux_date'] = to_datetime(expected['aux_date'],
dayfirst=True)
expected['aux_date'] = map(Timestamp, expected['aux_date'])
self.assert_(isinstance(expected['aux_date'][0], datetime))
df = read_csv(StringIO(data), sep=";", index_col = range(4),
parse_dates=[0, 5], dayfirst=True)
assert_frame_equal(df, expected)
df = read_csv(StringIO(data), sep=";", index_col = range(4),
parse_dates=['date', 'aux_date'], dayfirst=True)
assert_frame_equal(df, expected)
def test_parse_dates_column_list(self):
from pandas.core.datetools import to_datetime
data = '''date;destination;ventilationcode;unitcode;units;aux_date
01/01/2010;P;P;50;1;12/1/2011
01/01/2010;P;R;50;1;13/1/2011
15/01/2010;P;P;50;1;14/1/2011
01/05/2010;P;P;50;1;15/1/2011'''
expected = self.read_csv(StringIO(data), sep=";", index_col=lrange(4))
lev = expected.index.levels[0]
levels = list(expected.index.levels)
levels[0] = lev.to_datetime(dayfirst=True)
# hack to get this to work - remove for final test
levels[0].name = lev.name
expected.index.set_levels(levels, inplace=True)
expected['aux_date'] = to_datetime(expected['aux_date'],
dayfirst=True)
expected['aux_date'] = lmap(Timestamp, expected['aux_date'])
tm.assertIsInstance(expected['aux_date'][0], datetime)
df = self.read_csv(StringIO(data), sep=";", index_col=lrange(4),
parse_dates=[0, 5], dayfirst=True)
tm.assert_frame_equal(df, expected)
df = self.read_csv(StringIO(data), sep=";", index_col=lrange(4),
parse_dates=['date', 'aux_date'], dayfirst=True)
tm.assert_frame_equal(df, expected)
def __new__(cls, start=None, end=None, periods=None,
offset=datetools.bday, timeRule=None,
tzinfo=None, **kwds):
if timeRule is not None:
offset = datetools.getOffset(timeRule)
if timeRule is None:
if offset in datetools._offsetNames:
timeRule = datetools._offsetNames[offset]
# Cachable
if not start:
start = kwds.get('begin')
if not periods:
periods = kwds.get('nPeriods')
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
# inside cache range. Handle UTC case
useCache = (offset.isAnchored() and
isinstance(offset, datetools.CacheableOffset))
start, end, tzinfo = _figure_out_timezone(start, end, tzinfo)
useCache = useCache and _naive_in_cache_range(start, end)
if useCache:
index = cls._cached_range(start, end, periods=periods,
offset=offset, timeRule=timeRule)
if tzinfo is None:
return index
else:
xdr = generate_range(start=start, end=end, periods=periods,
offset=offset, timeRule=timeRule)
index = list(xdr)
if tzinfo is not None:
index = [d.replace(tzinfo=tzinfo) for d in index]
index = np.array(index, dtype=object, copy=False)
index = index.view(cls)
index.offset = offset
index.tzinfo = tzinfo
return index
def _parse_datetime(arg, global_ns={}, local_ns={}):
# If arg is an int smaller than 1e10 then it is likely a date
if isinstance(arg, int) and arg < 30000000:
return datetime(arg / 10000, (arg / 100) % 100, arg % 100)
# Attempt to use the standard datetime parsing
parsed = datetools.to_datetime(arg)
if isinstance(parsed, datetime):
return parsed
# if the parsing failed, try to evaluate the string
try:
dt = eval(arg, global_ns, local_ns)
return datetools.to_datetime(dt)
except NameError:
pass
raise TypeError("Supplied parameter value '%s' can not be converted to datetime." % arg)
def truncate(self, before=None, after=None):
"""Function truncate a sorted DataFrame / Series before and/or after
some particular dates.
Parameters
----------
before : date
Truncate before date
after : date
Truncate after date
Returns
-------
truncated : type of caller
"""
before = datetools.to_datetime(before)
after = datetools.to_datetime(after)
# returns view, want to copy
return self.ix[before:after].copy()
def truncate(self, before=None, after=None):
"""Function truncate a sorted DataFrame / Series before and/or after
some particular dates.
Parameters
----------
before : date
Truncate before date
after : date
Truncate after date
Returns
-------
truncated : type of caller
"""
before = datetools.to_datetime(before)
after = datetools.to_datetime(after)
# returns view, want to copy
return self.ix[before:after].copy()
def truncate(self, before=None, after=None):
"""Function truncate a sorted TimeSeries before and/or after
some particular dates.
Parameters
----------
before : date
Truncate before date
after : date
Truncate after date
Notes
-----
If TimeSeries is contained in a DataFrame, consider using the version
of the function there.
Returns
-------
TimeSeries
"""
before = datetools.to_datetime(before)
after = datetools.to_datetime(after)
if before is None:
beg_slice = 0
elif before in self.index:
beg_slice = self.index.indexMap[before]
elif before < self.index[-1]:
beg_slice = self.index.searchsorted(before, side='left')
else:
return Series([], index=NULL_INDEX)
if after is None:
end_slice = len(self)
elif after in self.index:
end_slice = self.index.indexMap[after] + 1
elif after > self.index[0]:
end_slice = self.index.searchsorted(after, side='right')
else:
return Series([], index=NULL_INDEX)
return self[beg_slice:end_slice]
def truncate(self, before=None, after=None):
"""Function truncate a sorted TimeSeries before and/or after
some particular dates.
Parameters
----------
before : date
Truncate before date
after : date
Truncate after date
Notes
-----
If TimeSeries is contained in a DataFrame, consider using the version
of the function there.
Returns
-------
TimeSeries
"""
before = datetools.to_datetime(before)
after = datetools.to_datetime(after)
if before is None:
beg_slice = 0
elif before in self.index:
beg_slice = self.index.indexMap[before]
elif before < self.index[-1]:
beg_slice = self.index.searchsorted(before, side='left')
else:
return Series([], index=NULL_INDEX)
if after is None:
end_slice = len(self)
elif after in self.index:
end_slice = self.index.indexMap[after] + 1
elif after > self.index[0]:
end_slice = self.index.searchsorted(after, side='right')
else:
return Series([], index=NULL_INDEX)
return self[beg_slice:end_slice]
def _parse_datetime(arg, global_ns={}, local_ns={}):
# If arg is an int smaller than 1e10 then it is likely a date
if isinstance(arg, int) and arg < 30000000:
return datetime(arg / 10000, (arg / 100) % 100, arg % 100)
# Attempt to use the standard datetime parsing
parsed = datetools.to_datetime(arg)
if isinstance(parsed, datetime):
return parsed
# if the parsing failed, try to evaluate the string
try:
dt = eval(arg, global_ns, local_ns)
return datetools.to_datetime(dt)
except NameError:
pass
raise TypeError(
"Supplied parameter value '%s' can not be converted to datetime." %
arg)
def __init__(self,
start=None,
end=None,
nPeriods=None,
offset=datetools.BDay(),
timeRule=None):
if timeRule is not None:
offset = datetools.getOffset(timeRule)
if timeRule is None:
if offset in datetools._offsetNames:
timeRule = datetools._offsetNames[offset]
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
if start and not offset.onOffset(start):
start = start + offset.__class__(n=1, **offset.kwds)
if end and not offset.onOffset(end):
end = end - offset.__class__(n=1, **offset.kwds)
if nPeriods == None and end < start:
end = None
nPeriods = 0
if end is None:
end = start + (nPeriods - 1) * offset
if start is None:
start = end - (nPeriods - 1) * offset
self.offset = offset
self.timeRule = timeRule
self.start = start
self.end = end
self.nPeriods = nPeriods
def asOf(self, date):
"""
Return last good (non-NaN) value in TimeSeries if value is NaN for
requested date.
If there is no good value, NaN is returned.
Parameters
----------
date : datetime or similar value
Notes
-----
Dates are assumed to be sorted
Returns
-------
value or NaN
"""
if isinstance(date, basestring):
date = datetools.to_datetime(date)
v = self.get(date)
if isnull(v):
candidates = self.index[notnull(self)]
index = candidates.searchsorted(date)
if index > 0:
asOfDate = candidates[index - 1]
else:
return NaN
return self.get(asOfDate)
else:
return v
def asOf(self, date):
"""
Return last good (non-NaN) value in TimeSeries if value is NaN for
requested date.
If there is no good value, NaN is returned.
Parameters
----------
date : datetime or similar value
Notes
-----
Dates are assumed to be sorted
Returns
-------
value or NaN
"""
if isinstance(date, basestring):
date = datetools.to_datetime(date)
v = self.get(date)
if isnull(v):
candidates = self.index[notnull(self)]
index = candidates.searchsorted(date)
if index > 0:
asOfDate = candidates[index - 1]
else:
return NaN
return self.get(asOfDate)
else:
return v
def generate_range(start=None, end=None, periods=None,
offset=datetools.BDay(), time_rule=None):
"""
Generates a sequence of dates corresponding to the specified time
offset. Similar to dateutil.rrule except uses pandas DateOffset
objects to represent time increments
Parameters
----------
start : datetime (default None)
end : datetime (default None)
periods : int, optional
Note
----
* This method is faster for generating weekdays than dateutil.rrule
* At least two of (start, end, periods) must be specified.
* If both start and end are specified, the returned dates will
satisfy start <= date <= end.
Returns
-------
dates : generator object
See also
--------
DateRange, dateutil.rrule
"""
if time_rule is not None:
offset = datetools.getOffset(time_rule)
if time_rule is None:
if offset in datetools._offsetNames:
time_rule = datetools._offsetNames[offset]
start = datetools.to_datetime(start)
end = datetools.to_datetime(end)
if start and not offset.onOffset(start):
start = offset.rollforward(start)
if end and not offset.onOffset(end):
end = offset.rollback(end)
if periods is None and end < start:
end = None
periods = 0
if end is None:
end = start + (periods - 1) * offset
if start is None:
start = end - (periods - 1) * offset
cur = start
if offset._normalizeFirst:
cur = datetools.normalize_date(cur)
while cur <= end:
yield cur
# faster than cur + offset
cur = offset.apply(cur)
def test_index_to_datetime(self):
idx = Index(["1/1/2000", "1/2/2000", "1/3/2000"])
result = idx.to_datetime()
expected = DatetimeIndex(datetools.to_datetime(idx.values))
self.assert_(result.equals(expected))
def update(self):
'''
Plots the 3D bars and axis.
:return:
'''
is_first_update = False
if self._last_view is None:
self._last_view = (38, 8, 205, array([ 8, 17.5, 49.25]))
self.scene.mlab.view(*self._last_view)
is_first_update = True
else:
self._last_view = self.scene.mlab.view()
self.scene.disable_render = True
self.clear_figure()
#print "Day: %s" % time.ctime(self.current_time)
max_z = 0
time_index = ((self._data_times <= datetools.to_datetime(self.current_time)) &
(self._data_times >= (datetools.to_datetime(self.current_time) - self._num_of_shown_days_to_timedelta())))
if self.selected_source is None: # Plot all sources
x = []
y = []
z = []
severities = []
for source in range(self._num_of_sources):
for data_index in array(range(len(self._data)))[time_index][self._data_sources[time_index] == source]:
if self.used_cache_size > 0 and self._cache.has_key(data_index):
devsptr, sevs, expectptr, min2, max2, count = self._cache[data_index]
else:
devs, sevs, expect, min2, max2= self._vis_model.calc_one(data_index)
devsptr = pyisc._to_cpp_array(devs)
expectptr = pyisc._to_cpp_array(expect)
count = None
if count is None:
self._trim_cache(data_index, self.used_cache_size)
vec = self._data._get_intfloat(data_index)
count = sum([pyisc._get_intfloat_value(vec, self._vis_model.get_event_hierarchy().get_index_value(l)) for l in range(self._vis_model.num_of_severity_levels_) if self._vis_model.get_event_hierarchy().get_index_value(l) != -1 ])
self._cache[data_index] = (devsptr, sevs, expectptr, min2, max2, count)
ztime = self._num_of_shown_days_to_int()-(self.current_time-self._data_times[data_index]).days
x.append(source)
y.append(ztime)
z.append(count)
sev_max = argmax(sevs)
sev = (-1 if sevs[sev_max] < self.anomaly_detection_threshold else sev_max)
severities.append(sev)
self._create_barcharts(severities, x, y, z)
max_z = max([max_z]+z)
else: # Plot for selected source
source_index = self._data_sources[time_index] == self.selected_source
data_indexes = array(range(len(self._data)))[time_index][source_index]
x = []
y = []
z = []
severities = []
# Plot selected events
for data_index in data_indexes:
if self.used_cache_size > 0 and self._cache.has_key(data_index):
devsptr, sevs, expectptr, min2, max2, _ = self._cache[data_index]
else:
devs, sevs,expect, min2, max2 = self._vis_model.calc_one(data_index)
devsptr = pyisc._to_cpp_array(devs)
expectptr = pyisc._to_cpp_array(expect)
self._trim_cache(data_index, self.used_cache_size)
self._cache[data_index] = (devsptr, sevs, expectptr, min2, max2, None)
ztime = self._num_of_shown_days_to_int()-(datetools.to_datetime(self.current_time)-self._data_times[data_index]).days
if self._vis_model.get_num_of_selected_events() > 0:
for element in range(self._vis_model.get_num_of_selected_events()):
x.append(element)
y.append(ztime)
(dev, sev, count, mexp, maxind) = self._vis_model.summarize_event_children(self._vis_model.get_selected_event(element), devsptr, expectptr, self._data._get_intfloat(data_index), 1 if element >= self.selected_event else 0)
z.append(count)
if dev < self.anomaly_detection_threshold:
severities.append(-1)
else:
severities.append(sev)
self._create_barcharts(severities, x, y, z)
max_z = max([max_z]+z)
self.scene.disable_render = True
datetime.date
curr_t = self.current_time
time_strs = [str(t.date())for t in date_range(curr_t-self._num_of_shown_days_to_timedelta(), curr_t)]
time_max_len = min([len(t) for t in time_strs])
max_x = (self._num_of_sources if self.selected_source is None else self._vis_model.get_num_of_selected_events())
max_y = self._num_of_shown_days_to_int()
if len(self.time_text3ds) != len(time_strs):
self.scene.remove_actors([t.actor.actors[0] for t in self.time_text3ds])
self.time_text3ds = []
for slot in range(len(time_strs)):
name = time_strs[slot]
pos = (max_x+time_max_len/2-1, slot, 0)
self.time_text3ds.append(self.scene.mlab.text3d(*pos, text=name, scale=0.5, color=self.textcolor, orient_to_camera=False, orientation=(180, 180, 0)))
else:
for slot in range(len(time_strs)):
name = time_strs[slot]
pos = (max_x+time_max_len/2-1, slot, 0)
self.time_text3ds[slot].position = pos
self.time_text3ds[slot].text=name
if self.selected_source is None:
source_strs = [self._get_source_name(source) for source in range(self._num_of_sources)]
num_of_sources = self._num_of_sources
else:
source_strs = [self._get_event_name(element) for element in range(self._vis_model.get_num_of_selected_events())]
num_of_sources = self._vis_model.get_num_of_selected_events()
if len(self.source_text3ds) != num_of_sources or self.selected_source is None:
self.scene.remove_actors([t.actor.actors[0] for t in self.source_text3ds])
self.source_text3ds = []
for source in range(num_of_sources):
name = source_strs[source]
if self.selected_source is None:
self.source_text3ds.append(self.scene.mlab.text3d(source, max_y + 0.5, 0, name, scale=0.6, color=self.textcolor, orient_to_camera=False, orientation=(0, 0, 90)))
else:
self.source_text3ds.append(self.scene.mlab.text3d(source, max_y + 0.5, 0, name, color=self.textcolor if source < self.selected_event else (192.0/255, 192.0/255, 192.0/255) if source > self.selected_event else (1.0, 1.0, 1.0), scale=0.5, orient_to_camera=False, orientation=(0, 0, 90)))
else:
for source in range(num_of_sources):
name = source_strs[source]
self.source_text3ds[source].text = name
self.source_text3ds[source].position = (source, max_y + 0.5, 0)
if is_first_update:
self.scene.reset_zoom()
self.scene.disable_render = False
return
def __init__(self, visualisation_model, decision_threshold, start_day=3, num_of_shown_days="30 days", precompute_cache=False):
'''
:param visualisation_model: an instance of EventDataModel
:param decision_threshold: a float larger or equal to 0.0 that is used for deciding when an anomaly score is significantly anomalous
:param start_day: an integer >= or an instance of datetime.date or an string, like "2014-10-11" or a tuple, like (2014, 10, 11)
:param num_of_shown_days: an integer > 1 that specifies the number of days back in time from start_day that will be shown.
:param precompute_cache: boolean that indates whether all anomaly scores should be computed at once or when asked for.
:return:
'''
assert isinstance(visualisation_model, EventDataModel)
assert isinstance(start_day, int) or isinstance(start_day, str) or isinstance(start_day, datetime.date) or (isinstance(start_day, tuple) and len(start_day) == 3)
HasTraits.__init__(self)
self.used_cache_size = 0 # must be initialized
self._data = visualisation_model._event_data_object
self.num_of_shown_days = num_of_shown_days # Updates self.used_cache_size
self._vis_model = visualisation_model
self._anomaly_detector = visualisation_model._anomaly_detector
self.anomaly_detection_threshold = decision_threshold
dates = visualisation_model._event_data_object.dates_
self._data_times = array([datetools.to_datetime(d) for d in dates])
self.source_names = list(unique(visualisation_model._event_data_object.sources_))
self._data_sources = array([self.source_names.index(source) for source in visualisation_model._event_data_object.sources_])
self._num_of_sources = len(unique(self.source_names)) # number of sources
self.barcharts = []
self.barchart_actors = []
self.time_text3ds = []
self.source_text3ds = []
self.xy_positions = []
self._high_start_day_number = int((self._data_times.max() - self._data_times.min()).days)
self.scene.anti_aliasing_frames = 8
# add traits dynamically
self.add_trait("Relative_Start_Day", Range(0, self._high_start_day_number))
self.add_trait("_selected_source_name", Enum(None,[None]+self.source_names))
self.configure_traits()
self.scene.background =self.background
# add the mouse pick handler
self.picker = self.scene.mayavi_scene.on_mouse_pick(self.vis_picker, 'cell')
self.picker.tolerance = 0.01
#cmap = matplotlib.cm.get_cmap('Reds')
#self.severity_color = [cmap(x)[:-1] for x in linspace(0.75, 0.95, self._vis_model.num_of_severity_levels_)]
if self._vis_model.num_of_severity_levels_ > 1:
self.severity_color = self.severity_color = [(1,x/100.0, x/100.0) for x in range(70, 30, -40/self._vis_model.num_of_severity_levels_)]
else:
self.severity_color = [(255.0 / 255, 51 / 255.0, 51 / 255.0)]
# This used for a fix to manage a bug in Mayavi library, an invisible default object
self._obj = self.scene.mlab.points3d(0, 0, 0, opacity=0.0)
# Cache all anomaly calculations for all data values
if precompute_cache:
self.used_cache_size = len(self._data)
for data_index in xrange(len(self._data)):
self._populate_cache(data_index)
self.start_day = start_day
self.update()
def update(self):
'''
Plots the 3D bars and axis.
:return:
'''
is_first_update = False
if self._last_view is None:
self._last_view = (38, 8, 205, array([8, 17.5, 49.25]))
self.scene.mlab.view(*self._last_view)
is_first_update = True
else:
self._last_view = self.scene.mlab.view()
self.scene.disable_render = True
self.clear_figure()
#print "Day: %s" % time.ctime(self.current_time)
max_z = 0
time_index = (
(self._data_times <= datetools.to_datetime(self.current_time)) &
(self._data_times >= (datetools.to_datetime(self.current_time) -
self._num_of_shown_days_to_timedelta())))
if self.selected_source is None: # Plot all sources
x = []
y = []
z = []
severities = []
for source in range(self._num_of_sources):
for data_index in array(range(len(self._data)))[time_index][
self._data_sources[time_index] == source]:
if self.used_cache_size > 0 and self._cache.has_key(
data_index):
devsptr, sevs, expectptr, min2, max2, count = self._cache[
data_index]
else:
devs, sevs, expect, min2, max2 = self._vis_model.calc_one(
data_index)
devsptr = pyisc._to_cpp_array(devs)
expectptr = pyisc._to_cpp_array(expect)
count = None
if count is None:
self._trim_cache(data_index, self.used_cache_size)
vec = self._data._get_intfloat(data_index)
count = sum([
pyisc._get_intfloat_value(
vec,
self._vis_model.get_event_hierarchy().
get_index_value(l)) for l in range(
self._vis_model.num_of_severity_levels_)
if self._vis_model.get_event_hierarchy().
get_index_value(l) != -1
])
self._cache[data_index] = (devsptr, sevs, expectptr,
min2, max2, count)
ztime = self._num_of_shown_days_to_int() - (
self.current_time - self._data_times[data_index]).days
x.append(source)
y.append(ztime)
z.append(count)
sev_max = argmax(sevs)
sev = (-1
if sevs[sev_max] < self.anomaly_detection_threshold
else sev_max)
severities.append(sev)
self._create_barcharts(severities, x, y, z)
max_z = max([max_z] + z)
else: # Plot for selected source
source_index = self._data_sources[
time_index] == self.selected_source
data_indexes = array(range(len(
self._data)))[time_index][source_index]
x = []
y = []
z = []
severities = []
# Plot selected events
for data_index in data_indexes:
if self.used_cache_size > 0 and self._cache.has_key(
data_index):
devsptr, sevs, expectptr, min2, max2, _ = self._cache[
data_index]
else:
devs, sevs, expect, min2, max2 = self._vis_model.calc_one(
data_index)
devsptr = pyisc._to_cpp_array(devs)
expectptr = pyisc._to_cpp_array(expect)
self._trim_cache(data_index, self.used_cache_size)
self._cache[data_index] = (devsptr, sevs, expectptr, min2,
max2, None)
ztime = self._num_of_shown_days_to_int() - (
datetools.to_datetime(self.current_time) -
self._data_times[data_index]).days
if self._vis_model.get_num_of_selected_events() > 0:
for element in range(
self._vis_model.get_num_of_selected_events()):
x.append(element)
y.append(ztime)
(dev, sev, count, mexp,
maxind) = self._vis_model.summarize_event_children(
self._vis_model.get_selected_event(element),
devsptr, expectptr,
self._data._get_intfloat(data_index),
1 if element >= self.selected_event else 0)
z.append(count)
if dev < self.anomaly_detection_threshold:
severities.append(-1)
else:
severities.append(sev)
self._create_barcharts(severities, x, y, z)
max_z = max([max_z] + z)
self.scene.disable_render = True
datetime.date
curr_t = self.current_time
time_strs = [
str(t.date()) for t in date_range(
curr_t - self._num_of_shown_days_to_timedelta(), curr_t)
]
time_max_len = min([len(t) for t in time_strs])
max_x = (self._num_of_sources if self.selected_source is None else
self._vis_model.get_num_of_selected_events())
max_y = self._num_of_shown_days_to_int()
if len(self.time_text3ds) != len(time_strs):
self.scene.remove_actors(
[t.actor.actors[0] for t in self.time_text3ds])
self.time_text3ds = []
for slot in range(len(time_strs)):
name = time_strs[slot]
pos = (max_x + time_max_len / 2 - 1, slot, 0)
self.time_text3ds.append(
self.scene.mlab.text3d(*pos,
text=name,
scale=0.5,
color=self.textcolor,
orient_to_camera=False,
orientation=(180, 180, 0)))
else:
for slot in range(len(time_strs)):
name = time_strs[slot]
pos = (max_x + time_max_len / 2 - 1, slot, 0)
self.time_text3ds[slot].position = pos
self.time_text3ds[slot].text = name
if self.selected_source is None:
source_strs = [
self._get_source_name(source)
for source in range(self._num_of_sources)
]
num_of_sources = self._num_of_sources
else:
source_strs = [
self._get_event_name(element) for element in range(
self._vis_model.get_num_of_selected_events())
]
num_of_sources = self._vis_model.get_num_of_selected_events()
if len(self.source_text3ds
) != num_of_sources or self.selected_source is None:
self.scene.remove_actors(
[t.actor.actors[0] for t in self.source_text3ds])
self.source_text3ds = []
for source in range(num_of_sources):
name = source_strs[source]
if self.selected_source is None:
self.source_text3ds.append(
self.scene.mlab.text3d(source,
max_y + 0.5,
0,
name,
scale=0.6,
color=self.textcolor,
orient_to_camera=False,
orientation=(0, 0, 90)))
else:
self.source_text3ds.append(
self.scene.mlab.text3d(
source,
max_y + 0.5,
0,
name,
color=self.textcolor
if source < self.selected_event else
(192.0 / 255, 192.0 / 255,
192.0 / 255) if source > self.selected_event else
(1.0, 1.0, 1.0),
scale=0.5,
orient_to_camera=False,
orientation=(0, 0, 90)))
else:
for source in range(num_of_sources):
name = source_strs[source]
self.source_text3ds[source].text = name
self.source_text3ds[source].position = (source, max_y + 0.5, 0)
if is_first_update:
self.scene.reset_zoom()
self.scene.disable_render = False
return
def __init__(self,
visualisation_model,
decision_threshold,
start_day=3,
num_of_shown_days="30 days",
precompute_cache=False):
'''
:param visualisation_model: an instance of EventDataModel
:param decision_threshold: a float larger or equal to 0.0 that is used for deciding when an anomaly score is significantly anomalous
:param start_day: an integer >= or an instance of datetime.date or an string, like "2014-10-11" or a tuple, like (2014, 10, 11)
:param num_of_shown_days: an integer > 1 that specifies the number of days back in time from start_day that will be shown.
:param precompute_cache: boolean that indates whether all anomaly scores should be computed at once or when asked for.
:return:
'''
assert isinstance(visualisation_model, EventDataModel)
assert isinstance(
start_day, int) or isinstance(start_day, str) or isinstance(
start_day, datetime.date) or (isinstance(start_day, tuple)
and len(start_day) == 3)
HasTraits.__init__(self)
self.used_cache_size = 0 # must be initialized
self._data = visualisation_model._event_data_object
self.num_of_shown_days = num_of_shown_days # Updates self.used_cache_size
self._vis_model = visualisation_model
self._anomaly_detector = visualisation_model._anomaly_detector
self.anomaly_detection_threshold = decision_threshold
dates = visualisation_model._event_data_object.dates_
self._data_times = array([datetools.to_datetime(d) for d in dates])
self.source_names = list(
unique(visualisation_model._event_data_object.sources_))
self._data_sources = array([
self.source_names.index(source)
for source in visualisation_model._event_data_object.sources_
])
self._num_of_sources = len(unique(
self.source_names)) # number of sources
self.barcharts = []
self.barchart_actors = []
self.time_text3ds = []
self.source_text3ds = []
self.xy_positions = []
self._high_start_day_number = int(
(self._data_times.max() - self._data_times.min()).days)
self.scene.anti_aliasing_frames = 8
# add traits dynamically
self.add_trait("Relative_Start_Day",
Range(0, self._high_start_day_number))
self.add_trait("_selected_source_name",
Enum(None, [None] + self.source_names))
self.configure_traits()
self.scene.background = self.background
# add the mouse pick handler
self.picker = self.scene.mayavi_scene.on_mouse_pick(
self.vis_picker, 'cell')
self.picker.tolerance = 0.01
#cmap = matplotlib.cm.get_cmap('Reds')
#self.severity_color = [cmap(x)[:-1] for x in linspace(0.75, 0.95, self._vis_model.num_of_severity_levels_)]
if self._vis_model.num_of_severity_levels_ > 1:
self.severity_color = self.severity_color = [
(1, x / 100.0, x / 100.0)
for x in range(70, 30, -40 /
self._vis_model.num_of_severity_levels_)
]
else:
self.severity_color = [(255.0 / 255, 51 / 255.0, 51 / 255.0)]
# This used for a fix to manage a bug in Mayavi library, an invisible default object
self._obj = self.scene.mlab.points3d(0, 0, 0, opacity=0.0)
# Cache all anomaly calculations for all data values
if precompute_cache:
self.used_cache_size = len(self._data)
for data_index in xrange(len(self._data)):
self._populate_cache(data_index)
self.start_day = start_day
self.update()