Пример #1
0
def truncate(self, before=None, after=None, copy=True):
    """Function truncate a sorted DataFrame / Series before and/or after
    some particular dates.

    Parameters
    ----------
    before : date
        Truncate before date
    after : date
        Truncate after date

    Returns
    -------
    truncated : type of caller
    """
    before = datetools.to_datetime(before)
    after = datetools.to_datetime(after)

    if before is not None and after is not None:
        assert (before <= after)

    left, right = self.index.slice_locs(before, after)
    result = self[left:right]

    if isinstance(self.index, MultiIndex):
        result.index = self.index.truncate(before, after)

    if copy:
        result = result.copy()

    return result
Пример #2
0
 def _current_time_changed(self, old_time, new_time):
     num_of_days = int((datetools.to_datetime(new_time) - self._data_times.min()).days)
     if self.Relative_Start_Day != num_of_days:
         self.Relative_Start_Day = num_of_days
     elif old_time != new_time:
         self.current_time = datetools.to_datetime(self.current_time)
         self.update()
Пример #3
0
def truncate(self, before=None, after=None, copy=True):
    """Function truncate a sorted DataFrame / Series before and/or after
    some particular dates.

    Parameters
    ----------
    before : date
        Truncate before date
    after : date
        Truncate after date

    Returns
    -------
    truncated : type of caller
    """
    before = datetools.to_datetime(before)
    after = datetools.to_datetime(after)

    if before is not None and after is not None:
        assert(before <= after)

    result = self.ix[before:after]

    if isinstance(self.index, MultiIndex):
        result.index = self.index.truncate(before, after)

    if copy:
        result = result.copy()

    return result
Пример #4
0
    def __init__(self, start=None, end=None, nPeriods=None,
                 offset=datetools.BDay(), timeRule=None):

        if timeRule is not None:
            offset = datetools.getOffset(timeRule)

        if timeRule is None:
            if offset in datetools._offsetNames:
                timeRule = datetools._offsetNames[offset]

        start = datetools.to_datetime(start)
        end = datetools.to_datetime(end)

        if start and not offset.onOffset(start):
            start = start + offset.__class__(n=1, **offset.kwds)
        if end and not offset.onOffset(end):
            end = end - offset.__class__(n=1, **offset.kwds)
            if nPeriods == None and end < start:
                end = None
                nPeriods = 0

        if end is None:
            end = start + (nPeriods - 1) * offset

        if start is None:
            start = end - (nPeriods - 1) * offset

        self.offset = offset
        self.timeRule = timeRule
        self.start = start
        self.end = end
        self.nPeriods = nPeriods
Пример #5
0
    def __new__(cls,
                start=None,
                end=None,
                periods=None,
                offset=datetools.bday,
                timeRule=None,
                **kwds):

        # Allow us to circumvent hitting the cache
        index = kwds.get('index')
        if index is None:
            if timeRule is not None:
                offset = datetools.getOffset(timeRule)

            if timeRule is None:
                if offset in datetools._offsetNames:
                    timeRule = datetools._offsetNames[offset]

            # Cachable
            if not start:
                start = kwds.get('begin')
            if not end:
                end = kwds.get('end')
            if not periods:
                periods = kwds.get('nPeriods')

            start = datetools.to_datetime(start)
            end = datetools.to_datetime(end)

            # inside cache range
            fromInside = start is not None and start > CACHE_START
            toInside = end is not None and end < CACHE_END

            useCache = fromInside and toInside

            if (useCache and offset.isAnchored()
                    and not isinstance(offset, datetools.Tick)):

                index = cls.getCachedRange(start,
                                           end,
                                           periods=periods,
                                           offset=offset,
                                           timeRule=timeRule)

            else:
                xdr = XDateRange(start=start,
                                 end=end,
                                 nPeriods=periods,
                                 offset=offset,
                                 timeRule=timeRule)

                index = np.array(list(xdr), dtype=object, copy=False)

                index = index.view(cls)
                index.offset = offset
        else:
            index = index.view(cls)

        return index
Пример #6
0
 def _sanitize_dates(self, start, end):
     start = to_datetime(start)
     end = to_datetime(end)
     if start is None:
         start = datetime.datetime.today() - datetime.timedelta(365)
     if end is None:
         end = datetime.datetime.today()
     return start, end
Пример #7
0
def _sanitize_dates(start, end):
    start = to_datetime(start)
    end = to_datetime(end)
    if start is None:
        start = datetime(1900, 1, 1)
    if end is None:
        end = datetime.today()
    return start, end
Пример #8
0
 def _current_time_changed(self, old_time, new_time):
     num_of_days = int(
         (datetools.to_datetime(new_time) - self._data_times.min()).days)
     if self.Relative_Start_Day != num_of_days:
         self.Relative_Start_Day = num_of_days
     elif old_time != new_time:
         self.current_time = datetools.to_datetime(self.current_time)
         self.update()
Пример #9
0
 def _sanitize_dates(self, start, end):
     start = to_datetime(start)
     end = to_datetime(end)
     if start is None:
         start = datetime.datetime.today() - datetime.timedelta(365)
     if end is None:
         end = datetime.datetime.today()
     return start, end
Пример #10
0
def _sanitize_dates(start_date, end_date):
    from pandas.core.datetools import to_datetime
    start_date = to_datetime(start_date)
    end_date = to_datetime(end_date)
    if start_date is None:
        start_date = DATETIME_START_DEFAULT
    if end_date is None:
        end_date = datetime.datetime.today()
    return start_date, end_date
Пример #11
0
def _sanitize_dates(start, end):
    from pandas.core.datetools import to_datetime
    start = to_datetime(start)
    end = to_datetime(end)
    if start is None:
        start = dt.datetime(2010, 1, 1)
    if end is None:
        end = dt.datetime.today()
    return start, end
Пример #12
0
def _sanitize_dates(start, end):
    from pandas.core.datetools import to_datetime
    start = to_datetime(start)
    end = to_datetime(end)
    if start is None:
        start = dt.datetime.today() - dt.timedelta(365)
    if end is None:
        end = dt.datetime.today()
    return start, end
Пример #13
0
 def _start_day_changed ( self, new_time ):
     if isinstance(new_time, int):
         self.current_time = datetools.to_datetime(self._data_times.min()+datetools.Day(new_time))
     elif isinstance(new_time, str):
         self.current_time = datetools.to_datetime(new_time)
     elif isinstance(new_time, datetime.date):
         self.current_time = datetools.to_datetime(new_time)
     else:
         print "Unsupported start day ", new_time
         return
Пример #14
0
def test_to_datetime1():
    actual = to_datetime(datetime(2008, 1, 15))
    assert actual == datetime(2008, 1, 15)

    actual = to_datetime('20080115')
    assert actual == datetime(2008, 1, 15)

    # unparseable
    s = 'Month 1, 1999'
    assert to_datetime(s) == s
Пример #15
0
def test_to_datetime1():
    actual = to_datetime(datetime(2008, 1, 15))
    assert actual == datetime(2008, 1, 15)

    actual = to_datetime('20080115')
    assert actual == datetime(2008, 1, 15)

    # unparseable
    s = 'Month 1, 1999'
    assert to_datetime(s) == s
Пример #16
0
def test_to_datetime1():
    actual = to_datetime(datetime(2008, 1, 15))
    assert actual == datetime(2008, 1, 15)

    actual = to_datetime("20080115")
    assert actual == datetime(2008, 1, 15)

    # unparseable
    s = "Month 1, 1999"
    assert to_datetime(s) == s
Пример #17
0
 def _start_day_changed(self, new_time):
     if isinstance(new_time, int):
         self.current_time = datetools.to_datetime(self._data_times.min() +
                                                   datetools.Day(new_time))
     elif isinstance(new_time, str):
         self.current_time = datetools.to_datetime(new_time)
     elif isinstance(new_time, datetime.date):
         self.current_time = datetools.to_datetime(new_time)
     else:
         print "Unsupported start day ", new_time
         return
Пример #18
0
    def __new__(cls, start=None, end=None, periods=None,
                offset=datetools.bday, time_rule=None,
                tzinfo=None, name=None, **kwds):

        time_rule = kwds.get('timeRule', time_rule)
        if time_rule is not None:
            offset = datetools.getOffset(time_rule)

        if time_rule is None:
            if offset in datetools._offsetNames:
                time_rule = datetools._offsetNames[offset]

        # Cachable
        if not start:
            start = kwds.get('begin')
        if not periods:
            periods = kwds.get('nPeriods')

        start = datetools.to_datetime(start)
        end = datetools.to_datetime(end)

        if start is not None and not isinstance(start, datetime):
            raise ValueError('Failed to convert %s to datetime' % start)

        if end is not None and not isinstance(end, datetime):
            raise ValueError('Failed to convert %s to datetime' % end)

        # inside cache range. Handle UTC case
        useCache = _will_use_cache(offset)

        start, end, tzinfo = _figure_out_timezone(start, end, tzinfo)
        useCache = useCache and _naive_in_cache_range(start, end)

        if useCache:
            index = cls._cached_range(start, end, periods=periods,
                                      offset=offset, time_rule=time_rule,
                                      name=name)
            if tzinfo is None:
                return index
        else:
            xdr = generate_range(start=start, end=end, periods=periods,
                                 offset=offset, time_rule=time_rule)
            index = list(xdr)

        if tzinfo is not None:
            index = [d.replace(tzinfo=tzinfo) for d in index]

        index = np.array(index, dtype=object, copy=False)
        index = index.view(cls)
        index.name = name
        index.offset = offset
        index.tzinfo = tzinfo
        return index
def _sanitize_dates(start_date, end_date):
    """
    Sanitize dates (set to default)
        start_date - default: DATETIME_START_DEFAULT
        end_date - default: today
    """
    from pandas.core.datetools import to_datetime
    start_date = to_datetime(start_date)
    end_date = to_datetime(end_date)
    if start_date is None:
        start_date = DATETIME_START_DEFAULT
    if end_date is None:
        end_date = datetime.datetime.today()
    return start_date, end_date
Пример #20
0
def _sanitize_dates(start_date, end_date):
    """
    Sanitize dates (set to default)
        start_date - default: DATETIME_START_DEFAULT
        end_date - default: today
    """
    from pandas.core.datetools import to_datetime
    start_date = to_datetime(start_date)
    end_date = to_datetime(end_date)
    if start_date is None:
        start_date = DATETIME_START_DEFAULT
    if end_date is None:
        end_date = datetime.datetime.today()
    return start_date, end_date
Пример #21
0
    def __new__(cls, start=None, end=None, periods=None,
                offset=datetools.bday, timeRule=None, **kwds):

        # Allow us to circumvent hitting the cache
        index = kwds.get('index')
        if index is None:
            if timeRule is not None:
                offset = datetools.getOffset(timeRule)

            if timeRule is None:
                if offset in datetools._offsetNames:
                    timeRule = datetools._offsetNames[offset]

            # Cachable
            if not start:
                start = kwds.get('begin')
            if not end:
                end = kwds.get('end')
            if not periods:
                periods = kwds.get('nPeriods')

            start = datetools.to_datetime(start)
            end = datetools.to_datetime(end)

            # inside cache range
            fromInside = start is not None and start > CACHE_START
            toInside = end is not None and end < CACHE_END

            useCache = fromInside and toInside

            if (useCache and offset.isAnchored() and
                not isinstance(offset, datetools.Tick)):

                index = cls.getCachedRange(start, end, periods=periods,
                                           offset=offset, timeRule=timeRule)

            else:
                xdr = XDateRange(start=start, end=end,
                                 nPeriods=periods, offset=offset,
                                 timeRule=timeRule)

                index = np.array(list(xdr), dtype=object, copy=False)

                index = index.view(cls)
                index.offset = offset
        else:
            index = index.view(cls)

        return index
Пример #22
0
    def test_parse_dates_column_list(self):
        from pandas.core.datetools import to_datetime

        data = '''date;destination;ventilationcode;unitcode;units;aux_date
01/01/2010;P;P;50;1;12/1/2011
01/01/2010;P;R;50;1;13/1/2011
15/01/2010;P;P;50;1;14/1/2011
01/05/2010;P;P;50;1;15/1/2011'''

        expected = read_csv(StringIO(data), sep=";", index_col=range(4))

        lev = expected.index.levels[0]
        expected.index.levels[0] = lev.to_datetime(dayfirst=True)
        expected['aux_date'] = to_datetime(expected['aux_date'], dayfirst=True)
        expected['aux_date'] = map(Timestamp, expected['aux_date'])
        self.assert_(isinstance(expected['aux_date'][0], datetime))

        df = read_csv(StringIO(data),
                      sep=";",
                      index_col=range(4),
                      parse_dates=[0, 5],
                      dayfirst=True)
        assert_frame_equal(df, expected)

        df = read_csv(StringIO(data),
                      sep=";",
                      index_col=range(4),
                      parse_dates=['date', 'aux_date'],
                      dayfirst=True)
        assert_frame_equal(df, expected)
Пример #23
0
    def test_parse_dates_column_list(self):
        from pandas.core.datetools import to_datetime

        data = '''date;destination;ventilationcode;unitcode;units;aux_date
01/01/2010;P;P;50;1;12/1/2011
01/01/2010;P;R;50;1;13/1/2011
15/01/2010;P;P;50;1;14/1/2011
01/05/2010;P;P;50;1;15/1/2011'''

        expected = read_csv(StringIO(data), sep=";", index_col=range(4))

        lev = expected.index.levels[0]
        expected.index.levels[0] = lev.to_datetime(dayfirst=True)
        expected['aux_date'] = to_datetime(expected['aux_date'],
                                           dayfirst=True)
        expected['aux_date'] = map(Timestamp, expected['aux_date'])
        self.assert_(isinstance(expected['aux_date'][0], datetime))

        df = read_csv(StringIO(data), sep=";", index_col = range(4),
                      parse_dates=[0, 5], dayfirst=True)
        assert_frame_equal(df, expected)

        df = read_csv(StringIO(data), sep=";", index_col = range(4),
                      parse_dates=['date', 'aux_date'], dayfirst=True)
        assert_frame_equal(df, expected)
Пример #24
0
    def test_parse_dates_column_list(self):
        from pandas.core.datetools import to_datetime

        data = '''date;destination;ventilationcode;unitcode;units;aux_date
01/01/2010;P;P;50;1;12/1/2011
01/01/2010;P;R;50;1;13/1/2011
15/01/2010;P;P;50;1;14/1/2011
01/05/2010;P;P;50;1;15/1/2011'''

        expected = self.read_csv(StringIO(data), sep=";", index_col=lrange(4))

        lev = expected.index.levels[0]
        levels = list(expected.index.levels)
        levels[0] = lev.to_datetime(dayfirst=True)
        # hack to get this to work - remove for final test
        levels[0].name = lev.name
        expected.index.set_levels(levels, inplace=True)
        expected['aux_date'] = to_datetime(expected['aux_date'],
                                           dayfirst=True)
        expected['aux_date'] = lmap(Timestamp, expected['aux_date'])
        tm.assertIsInstance(expected['aux_date'][0], datetime)

        df = self.read_csv(StringIO(data), sep=";", index_col=lrange(4),
                           parse_dates=[0, 5], dayfirst=True)
        tm.assert_frame_equal(df, expected)

        df = self.read_csv(StringIO(data), sep=";", index_col=lrange(4),
                           parse_dates=['date', 'aux_date'], dayfirst=True)
        tm.assert_frame_equal(df, expected)
Пример #25
0
    def __new__(cls, start=None, end=None, periods=None,
                offset=datetools.bday, timeRule=None,
                tzinfo=None, **kwds):
        if timeRule is not None:
            offset = datetools.getOffset(timeRule)

        if timeRule is None:
            if offset in datetools._offsetNames:
                timeRule = datetools._offsetNames[offset]

        # Cachable
        if not start:
            start = kwds.get('begin')
        if not periods:
            periods = kwds.get('nPeriods')

        start = datetools.to_datetime(start)
        end = datetools.to_datetime(end)

        # inside cache range. Handle UTC case

        useCache = (offset.isAnchored() and
                    isinstance(offset, datetools.CacheableOffset))

        start, end, tzinfo = _figure_out_timezone(start, end, tzinfo)
        useCache = useCache and _naive_in_cache_range(start, end)

        if useCache:
            index = cls._cached_range(start, end, periods=periods,
                                      offset=offset, timeRule=timeRule)
            if tzinfo is None:
                return index
        else:
            xdr = generate_range(start=start, end=end, periods=periods,
                                 offset=offset, timeRule=timeRule)
            index = list(xdr)

        if tzinfo is not None:
            index = [d.replace(tzinfo=tzinfo) for d in index]

        index = np.array(index, dtype=object, copy=False)
        index = index.view(cls)
        index.offset = offset
        index.tzinfo = tzinfo
        return index
Пример #26
0
def _parse_datetime(arg, global_ns={}, local_ns={}):
    # If arg is an int smaller than 1e10 then it is likely a date
    if isinstance(arg, int) and arg < 30000000:
        return datetime(arg / 10000, (arg / 100) % 100, arg % 100)

    # Attempt to use the standard datetime parsing
    parsed = datetools.to_datetime(arg)
    if isinstance(parsed, datetime):
        return parsed

    # if the parsing failed, try to evaluate the string
    try:
        dt = eval(arg, global_ns, local_ns)
        return datetools.to_datetime(dt)
    except NameError:
        pass

    raise TypeError("Supplied parameter value '%s' can not be converted to datetime." % arg)
Пример #27
0
    def truncate(self, before=None, after=None):
        """Function truncate a sorted DataFrame / Series before and/or after
        some particular dates.

        Parameters
        ----------
        before : date
            Truncate before date
        after : date
            Truncate after date

        Returns
        -------
        truncated : type of caller
        """
        before = datetools.to_datetime(before)
        after = datetools.to_datetime(after)
        # returns view, want to copy
        return self.ix[before:after].copy()
Пример #28
0
    def truncate(self, before=None, after=None):
        """Function truncate a sorted DataFrame / Series before and/or after
        some particular dates.

        Parameters
        ----------
        before : date
            Truncate before date
        after : date
            Truncate after date

        Returns
        -------
        truncated : type of caller
        """
        before = datetools.to_datetime(before)
        after = datetools.to_datetime(after)
        # returns view, want to copy
        return self.ix[before:after].copy()
Пример #29
0
    def truncate(self, before=None, after=None):
        """Function truncate a sorted TimeSeries before and/or after
        some particular dates.

        Parameters
        ----------
        before : date
            Truncate before date
        after : date
            Truncate after date

        Notes
        -----
        If TimeSeries is contained in a DataFrame, consider using the version
        of the function there.

        Returns
        -------
        TimeSeries
        """
        before = datetools.to_datetime(before)
        after = datetools.to_datetime(after)

        if before is None:
            beg_slice = 0
        elif before in self.index:
            beg_slice = self.index.indexMap[before]
        elif before < self.index[-1]:
            beg_slice = self.index.searchsorted(before, side='left')
        else:
            return Series([], index=NULL_INDEX)

        if after is None:
            end_slice = len(self)
        elif after in self.index:
            end_slice = self.index.indexMap[after] + 1
        elif after > self.index[0]:
            end_slice = self.index.searchsorted(after, side='right')
        else:
            return Series([], index=NULL_INDEX)

        return self[beg_slice:end_slice]
Пример #30
0
    def truncate(self, before=None, after=None):
        """Function truncate a sorted TimeSeries before and/or after
        some particular dates.

        Parameters
        ----------
        before : date
            Truncate before date
        after : date
            Truncate after date

        Notes
        -----
        If TimeSeries is contained in a DataFrame, consider using the version
        of the function there.

        Returns
        -------
        TimeSeries
        """
        before = datetools.to_datetime(before)
        after = datetools.to_datetime(after)

        if before is None:
            beg_slice = 0
        elif before in self.index:
            beg_slice = self.index.indexMap[before]
        elif before < self.index[-1]:
            beg_slice = self.index.searchsorted(before, side='left')
        else:
            return Series([], index=NULL_INDEX)

        if after is None:
            end_slice = len(self)
        elif after in self.index:
            end_slice = self.index.indexMap[after] + 1
        elif after > self.index[0]:
            end_slice = self.index.searchsorted(after, side='right')
        else:
            return Series([], index=NULL_INDEX)

        return self[beg_slice:end_slice]
Пример #31
0
def _parse_datetime(arg, global_ns={}, local_ns={}):
    # If arg is an int smaller than 1e10 then it is likely a date
    if isinstance(arg, int) and arg < 30000000:
        return datetime(arg / 10000, (arg / 100) % 100, arg % 100)

    # Attempt to use the standard datetime parsing
    parsed = datetools.to_datetime(arg)
    if isinstance(parsed, datetime):
        return parsed

    # if the parsing failed, try to evaluate the string
    try:
        dt = eval(arg, global_ns, local_ns)
        return datetools.to_datetime(dt)
    except NameError:
        pass

    raise TypeError(
        "Supplied parameter value '%s' can not be converted to datetime." %
        arg)
Пример #32
0
    def __init__(self,
                 start=None,
                 end=None,
                 nPeriods=None,
                 offset=datetools.BDay(),
                 timeRule=None):

        if timeRule is not None:
            offset = datetools.getOffset(timeRule)

        if timeRule is None:
            if offset in datetools._offsetNames:
                timeRule = datetools._offsetNames[offset]

        start = datetools.to_datetime(start)
        end = datetools.to_datetime(end)

        if start and not offset.onOffset(start):
            start = start + offset.__class__(n=1, **offset.kwds)
        if end and not offset.onOffset(end):
            end = end - offset.__class__(n=1, **offset.kwds)
            if nPeriods == None and end < start:
                end = None
                nPeriods = 0

        if end is None:
            end = start + (nPeriods - 1) * offset

        if start is None:
            start = end - (nPeriods - 1) * offset

        self.offset = offset
        self.timeRule = timeRule
        self.start = start
        self.end = end
        self.nPeriods = nPeriods
Пример #33
0
    def asOf(self, date):
        """
        Return last good (non-NaN) value in TimeSeries if value is NaN for
        requested date.

        If there is no good value, NaN is returned.

        Parameters
        ----------
        date : datetime or similar value

        Notes
        -----
        Dates are assumed to be sorted

        Returns
        -------
        value or NaN
        """
        if isinstance(date, basestring):
            date = datetools.to_datetime(date)

        v = self.get(date)

        if isnull(v):
            candidates = self.index[notnull(self)]
            index = candidates.searchsorted(date)

            if index > 0:
                asOfDate = candidates[index - 1]
            else:
                return NaN

            return self.get(asOfDate)
        else:
            return v
Пример #34
0
    def asOf(self, date):
        """
        Return last good (non-NaN) value in TimeSeries if value is NaN for
        requested date.

        If there is no good value, NaN is returned.

        Parameters
        ----------
        date : datetime or similar value

        Notes
        -----
        Dates are assumed to be sorted

        Returns
        -------
        value or NaN
        """
        if isinstance(date, basestring):
            date = datetools.to_datetime(date)

        v = self.get(date)

        if isnull(v):
            candidates = self.index[notnull(self)]
            index = candidates.searchsorted(date)

            if index > 0:
                asOfDate = candidates[index - 1]
            else:
                return NaN

            return self.get(asOfDate)
        else:
            return v
Пример #35
0
def generate_range(start=None, end=None, periods=None,
                   offset=datetools.BDay(), time_rule=None):
    """
    Generates a sequence of dates corresponding to the specified time
    offset. Similar to dateutil.rrule except uses pandas DateOffset
    objects to represent time increments

    Parameters
    ----------
    start : datetime (default None)
    end : datetime (default None)
    periods : int, optional

    Note
    ----
    * This method is faster for generating weekdays than dateutil.rrule
    * At least two of (start, end, periods) must be specified.
    * If both start and end are specified, the returned dates will
    satisfy start <= date <= end.

    Returns
    -------
    dates : generator object

    See also
    --------
    DateRange, dateutil.rrule
    """

    if time_rule is not None:
        offset = datetools.getOffset(time_rule)

    if time_rule is None:
        if offset in datetools._offsetNames:
            time_rule = datetools._offsetNames[offset]

    start = datetools.to_datetime(start)
    end = datetools.to_datetime(end)

    if start and not offset.onOffset(start):
        start = offset.rollforward(start)

    if end and not offset.onOffset(end):
        end = offset.rollback(end)

        if periods is None and end < start:
            end = None
            periods = 0

    if end is None:
        end = start + (periods - 1) * offset

    if start is None:
        start = end - (periods - 1) * offset

    cur = start
    if offset._normalizeFirst:
        cur = datetools.normalize_date(cur)

    while cur <= end:
        yield cur

        # faster than cur + offset
        cur = offset.apply(cur)
Пример #36
0
    def test_index_to_datetime(self):
        idx = Index(["1/1/2000", "1/2/2000", "1/3/2000"])

        result = idx.to_datetime()
        expected = DatetimeIndex(datetools.to_datetime(idx.values))
        self.assert_(result.equals(expected))
Пример #37
0
    def update(self):
        '''
        Plots the 3D bars and axis.
        :return:
        '''
        is_first_update = False

        if self._last_view is None:
            self._last_view = (38, 8, 205, array([  8,  17.5,  49.25]))
            self.scene.mlab.view(*self._last_view)
            is_first_update = True
        else:
            self._last_view = self.scene.mlab.view()

        self.scene.disable_render = True
        self.clear_figure()
        #print "Day: %s" % time.ctime(self.current_time)


        max_z = 0

        time_index = ((self._data_times <= datetools.to_datetime(self.current_time)) &
                      (self._data_times >= (datetools.to_datetime(self.current_time) - self._num_of_shown_days_to_timedelta())))

        if self.selected_source is None: # Plot all sources
            x = []
            y = []
            z = []
            severities = []


            for source in range(self._num_of_sources):
                for data_index in array(range(len(self._data)))[time_index][self._data_sources[time_index] == source]:
                    if self.used_cache_size > 0 and self._cache.has_key(data_index):
                        devsptr, sevs, expectptr, min2, max2, count = self._cache[data_index]
                    else:
                        devs, sevs, expect, min2, max2= self._vis_model.calc_one(data_index)

                        devsptr = pyisc._to_cpp_array(devs)
                        expectptr = pyisc._to_cpp_array(expect)

                        count = None

                    if count is None:
                        self._trim_cache(data_index, self.used_cache_size)
                        vec = self._data._get_intfloat(data_index)
                        count = sum([pyisc._get_intfloat_value(vec, self._vis_model.get_event_hierarchy().get_index_value(l)) for l in range(self._vis_model.num_of_severity_levels_) if self._vis_model.get_event_hierarchy().get_index_value(l) != -1 ])
                        self._cache[data_index] = (devsptr, sevs, expectptr, min2, max2, count)


                    ztime = self._num_of_shown_days_to_int()-(self.current_time-self._data_times[data_index]).days

                    x.append(source)
                    y.append(ztime)

                    z.append(count)


                    sev_max = argmax(sevs)
                    sev = (-1 if sevs[sev_max] < self.anomaly_detection_threshold else sev_max)

                    severities.append(sev)
            self._create_barcharts(severities, x, y, z)

            max_z = max([max_z]+z)
        else: # Plot for selected source
            source_index = self._data_sources[time_index] == self.selected_source

            data_indexes = array(range(len(self._data)))[time_index][source_index]
            x = []
            y = []
            z = []
            severities = []

            # Plot selected events
            for data_index in data_indexes:
                if self.used_cache_size > 0 and self._cache.has_key(data_index):
                    devsptr, sevs, expectptr, min2, max2, _ = self._cache[data_index]
                else:
                    devs, sevs,expect, min2, max2 = self._vis_model.calc_one(data_index)
                    devsptr = pyisc._to_cpp_array(devs)
                    expectptr = pyisc._to_cpp_array(expect)


                    self._trim_cache(data_index, self.used_cache_size)

                    self._cache[data_index] = (devsptr, sevs, expectptr, min2, max2, None)

                ztime = self._num_of_shown_days_to_int()-(datetools.to_datetime(self.current_time)-self._data_times[data_index]).days


                if self._vis_model.get_num_of_selected_events() > 0:
                    for element in range(self._vis_model.get_num_of_selected_events()):
                        x.append(element)
                        y.append(ztime)

                        (dev, sev, count, mexp, maxind) = self._vis_model.summarize_event_children(self._vis_model.get_selected_event(element), devsptr, expectptr, self._data._get_intfloat(data_index), 1 if element >= self.selected_event else 0)
                        z.append(count)


                        if dev < self.anomaly_detection_threshold:
                            severities.append(-1)
                        else:
                            severities.append(sev)

            self._create_barcharts(severities, x, y, z)
            max_z = max([max_z]+z)

        self.scene.disable_render = True

        datetime.date

        curr_t = self.current_time
        time_strs = [str(t.date())for t in date_range(curr_t-self._num_of_shown_days_to_timedelta(), curr_t)]
        time_max_len = min([len(t) for t in time_strs])

        max_x = (self._num_of_sources if self.selected_source is None else  self._vis_model.get_num_of_selected_events())
        max_y = self._num_of_shown_days_to_int()

        if len(self.time_text3ds) != len(time_strs):
            self.scene.remove_actors([t.actor.actors[0] for t in self.time_text3ds])
            self.time_text3ds = []

            for slot in range(len(time_strs)):
                name = time_strs[slot]
                pos = (max_x+time_max_len/2-1, slot, 0)
                self.time_text3ds.append(self.scene.mlab.text3d(*pos, text=name, scale=0.5, color=self.textcolor, orient_to_camera=False, orientation=(180, 180, 0)))
        else:
            for slot in range(len(time_strs)):
                name = time_strs[slot]
                pos = (max_x+time_max_len/2-1, slot, 0)

                self.time_text3ds[slot].position = pos
                self.time_text3ds[slot].text=name

        if self.selected_source is None:
            source_strs = [self._get_source_name(source) for source in range(self._num_of_sources)]
            num_of_sources = self._num_of_sources
        else:
            source_strs = [self._get_event_name(element) for element in range(self._vis_model.get_num_of_selected_events())]
            num_of_sources = self._vis_model.get_num_of_selected_events()

        if len(self.source_text3ds) != num_of_sources or self.selected_source is None:
            self.scene.remove_actors([t.actor.actors[0] for t in self.source_text3ds])
            self.source_text3ds = []
            for source in range(num_of_sources):
                name = source_strs[source]
                if self.selected_source is None:
                    self.source_text3ds.append(self.scene.mlab.text3d(source, max_y + 0.5, 0, name, scale=0.6, color=self.textcolor, orient_to_camera=False, orientation=(0, 0, 90)))
                else:
                    self.source_text3ds.append(self.scene.mlab.text3d(source, max_y + 0.5, 0, name, color=self.textcolor if source < self.selected_event else (192.0/255, 192.0/255, 192.0/255) if source > self.selected_event else (1.0, 1.0,  1.0), scale=0.5, orient_to_camera=False, orientation=(0, 0, 90)))
        else:
            for source in range(num_of_sources):
                name = source_strs[source]
                self.source_text3ds[source].text = name
                self.source_text3ds[source].position = (source, max_y + 0.5, 0)

        if is_first_update:
            self.scene.reset_zoom()

        self.scene.disable_render = False

        return
Пример #38
0
    def __init__(self, visualisation_model, decision_threshold, start_day=3, num_of_shown_days="30 days", precompute_cache=False):
        '''

        :param visualisation_model: an instance of EventDataModel
        :param decision_threshold: a float larger or equal to 0.0 that is used for deciding when an anomaly score is significantly anomalous
        :param start_day: an integer >= or an instance of datetime.date or an string, like "2014-10-11" or a tuple, like (2014, 10, 11)
        :param num_of_shown_days: an integer > 1 that specifies the number of days back in time from start_day that will be shown.
        :param precompute_cache: boolean that indates whether all anomaly scores should be computed at once or when asked for.
        :return:
        '''
        assert isinstance(visualisation_model, EventDataModel)
        assert isinstance(start_day, int) or isinstance(start_day, str) or isinstance(start_day, datetime.date) or (isinstance(start_day, tuple) and len(start_day) == 3)
        HasTraits.__init__(self)

        self.used_cache_size = 0 # must be initialized
        self._data = visualisation_model._event_data_object

        self.num_of_shown_days = num_of_shown_days # Updates self.used_cache_size

        self._vis_model = visualisation_model
        self._anomaly_detector = visualisation_model._anomaly_detector
        self.anomaly_detection_threshold = decision_threshold

        dates = visualisation_model._event_data_object.dates_
        self._data_times = array([datetools.to_datetime(d) for d in dates])

        self.source_names = list(unique(visualisation_model._event_data_object.sources_))
        self._data_sources = array([self.source_names.index(source) for source in visualisation_model._event_data_object.sources_])
        self._num_of_sources = len(unique(self.source_names)) # number of sources

        self.barcharts = []
        self.barchart_actors = []
        self.time_text3ds = []
        self.source_text3ds = []
        self.xy_positions = []

        self._high_start_day_number = int((self._data_times.max() - self._data_times.min()).days)

        self.scene.anti_aliasing_frames = 8

        # add traits dynamically
        self.add_trait("Relative_Start_Day", Range(0, self._high_start_day_number))

        self.add_trait("_selected_source_name", Enum(None,[None]+self.source_names))

        self.configure_traits()

        self.scene.background =self.background

        # add the mouse pick handler
        self.picker = self.scene.mayavi_scene.on_mouse_pick(self.vis_picker, 'cell')

        self.picker.tolerance = 0.01

        #cmap = matplotlib.cm.get_cmap('Reds')
        #self.severity_color = [cmap(x)[:-1] for x in linspace(0.75, 0.95, self._vis_model.num_of_severity_levels_)]

        if self._vis_model.num_of_severity_levels_ > 1:
            self.severity_color = self.severity_color = [(1,x/100.0, x/100.0) for x in range(70, 30, -40/self._vis_model.num_of_severity_levels_)]
        else:
            self.severity_color = [(255.0 / 255, 51 / 255.0, 51 / 255.0)]


        # This used for a fix to manage a bug in Mayavi library, an invisible default object
        self._obj = self.scene.mlab.points3d(0, 0, 0, opacity=0.0)

        # Cache all anomaly calculations for all data values
        if precompute_cache:
            self.used_cache_size = len(self._data)
            for data_index in xrange(len(self._data)):
                self._populate_cache(data_index)

        self.start_day = start_day

        self.update()
Пример #39
0
    def update(self):
        '''
        Plots the 3D bars and axis.
        :return:
        '''
        is_first_update = False

        if self._last_view is None:
            self._last_view = (38, 8, 205, array([8, 17.5, 49.25]))
            self.scene.mlab.view(*self._last_view)
            is_first_update = True
        else:
            self._last_view = self.scene.mlab.view()

        self.scene.disable_render = True
        self.clear_figure()
        #print "Day: %s" % time.ctime(self.current_time)

        max_z = 0

        time_index = (
            (self._data_times <= datetools.to_datetime(self.current_time)) &
            (self._data_times >= (datetools.to_datetime(self.current_time) -
                                  self._num_of_shown_days_to_timedelta())))

        if self.selected_source is None:  # Plot all sources
            x = []
            y = []
            z = []
            severities = []

            for source in range(self._num_of_sources):
                for data_index in array(range(len(self._data)))[time_index][
                        self._data_sources[time_index] == source]:
                    if self.used_cache_size > 0 and self._cache.has_key(
                            data_index):
                        devsptr, sevs, expectptr, min2, max2, count = self._cache[
                            data_index]
                    else:
                        devs, sevs, expect, min2, max2 = self._vis_model.calc_one(
                            data_index)

                        devsptr = pyisc._to_cpp_array(devs)
                        expectptr = pyisc._to_cpp_array(expect)

                        count = None

                    if count is None:
                        self._trim_cache(data_index, self.used_cache_size)
                        vec = self._data._get_intfloat(data_index)
                        count = sum([
                            pyisc._get_intfloat_value(
                                vec,
                                self._vis_model.get_event_hierarchy().
                                get_index_value(l)) for l in range(
                                    self._vis_model.num_of_severity_levels_)
                            if self._vis_model.get_event_hierarchy().
                            get_index_value(l) != -1
                        ])
                        self._cache[data_index] = (devsptr, sevs, expectptr,
                                                   min2, max2, count)

                    ztime = self._num_of_shown_days_to_int() - (
                        self.current_time - self._data_times[data_index]).days

                    x.append(source)
                    y.append(ztime)

                    z.append(count)

                    sev_max = argmax(sevs)
                    sev = (-1
                           if sevs[sev_max] < self.anomaly_detection_threshold
                           else sev_max)

                    severities.append(sev)
            self._create_barcharts(severities, x, y, z)

            max_z = max([max_z] + z)
        else:  # Plot for selected source
            source_index = self._data_sources[
                time_index] == self.selected_source

            data_indexes = array(range(len(
                self._data)))[time_index][source_index]
            x = []
            y = []
            z = []
            severities = []

            # Plot selected events
            for data_index in data_indexes:
                if self.used_cache_size > 0 and self._cache.has_key(
                        data_index):
                    devsptr, sevs, expectptr, min2, max2, _ = self._cache[
                        data_index]
                else:
                    devs, sevs, expect, min2, max2 = self._vis_model.calc_one(
                        data_index)
                    devsptr = pyisc._to_cpp_array(devs)
                    expectptr = pyisc._to_cpp_array(expect)

                    self._trim_cache(data_index, self.used_cache_size)

                    self._cache[data_index] = (devsptr, sevs, expectptr, min2,
                                               max2, None)

                ztime = self._num_of_shown_days_to_int() - (
                    datetools.to_datetime(self.current_time) -
                    self._data_times[data_index]).days

                if self._vis_model.get_num_of_selected_events() > 0:
                    for element in range(
                            self._vis_model.get_num_of_selected_events()):
                        x.append(element)
                        y.append(ztime)

                        (dev, sev, count, mexp,
                         maxind) = self._vis_model.summarize_event_children(
                             self._vis_model.get_selected_event(element),
                             devsptr, expectptr,
                             self._data._get_intfloat(data_index),
                             1 if element >= self.selected_event else 0)
                        z.append(count)

                        if dev < self.anomaly_detection_threshold:
                            severities.append(-1)
                        else:
                            severities.append(sev)

            self._create_barcharts(severities, x, y, z)
            max_z = max([max_z] + z)

        self.scene.disable_render = True

        datetime.date

        curr_t = self.current_time
        time_strs = [
            str(t.date()) for t in date_range(
                curr_t - self._num_of_shown_days_to_timedelta(), curr_t)
        ]
        time_max_len = min([len(t) for t in time_strs])

        max_x = (self._num_of_sources if self.selected_source is None else
                 self._vis_model.get_num_of_selected_events())
        max_y = self._num_of_shown_days_to_int()

        if len(self.time_text3ds) != len(time_strs):
            self.scene.remove_actors(
                [t.actor.actors[0] for t in self.time_text3ds])
            self.time_text3ds = []

            for slot in range(len(time_strs)):
                name = time_strs[slot]
                pos = (max_x + time_max_len / 2 - 1, slot, 0)
                self.time_text3ds.append(
                    self.scene.mlab.text3d(*pos,
                                           text=name,
                                           scale=0.5,
                                           color=self.textcolor,
                                           orient_to_camera=False,
                                           orientation=(180, 180, 0)))
        else:
            for slot in range(len(time_strs)):
                name = time_strs[slot]
                pos = (max_x + time_max_len / 2 - 1, slot, 0)

                self.time_text3ds[slot].position = pos
                self.time_text3ds[slot].text = name

        if self.selected_source is None:
            source_strs = [
                self._get_source_name(source)
                for source in range(self._num_of_sources)
            ]
            num_of_sources = self._num_of_sources
        else:
            source_strs = [
                self._get_event_name(element) for element in range(
                    self._vis_model.get_num_of_selected_events())
            ]
            num_of_sources = self._vis_model.get_num_of_selected_events()

        if len(self.source_text3ds
               ) != num_of_sources or self.selected_source is None:
            self.scene.remove_actors(
                [t.actor.actors[0] for t in self.source_text3ds])
            self.source_text3ds = []
            for source in range(num_of_sources):
                name = source_strs[source]
                if self.selected_source is None:
                    self.source_text3ds.append(
                        self.scene.mlab.text3d(source,
                                               max_y + 0.5,
                                               0,
                                               name,
                                               scale=0.6,
                                               color=self.textcolor,
                                               orient_to_camera=False,
                                               orientation=(0, 0, 90)))
                else:
                    self.source_text3ds.append(
                        self.scene.mlab.text3d(
                            source,
                            max_y + 0.5,
                            0,
                            name,
                            color=self.textcolor
                            if source < self.selected_event else
                            (192.0 / 255, 192.0 / 255,
                             192.0 / 255) if source > self.selected_event else
                            (1.0, 1.0, 1.0),
                            scale=0.5,
                            orient_to_camera=False,
                            orientation=(0, 0, 90)))
        else:
            for source in range(num_of_sources):
                name = source_strs[source]
                self.source_text3ds[source].text = name
                self.source_text3ds[source].position = (source, max_y + 0.5, 0)

        if is_first_update:
            self.scene.reset_zoom()

        self.scene.disable_render = False

        return
Пример #40
0
    def __init__(self,
                 visualisation_model,
                 decision_threshold,
                 start_day=3,
                 num_of_shown_days="30 days",
                 precompute_cache=False):
        '''

        :param visualisation_model: an instance of EventDataModel
        :param decision_threshold: a float larger or equal to 0.0 that is used for deciding when an anomaly score is significantly anomalous
        :param start_day: an integer >= or an instance of datetime.date or an string, like "2014-10-11" or a tuple, like (2014, 10, 11)
        :param num_of_shown_days: an integer > 1 that specifies the number of days back in time from start_day that will be shown.
        :param precompute_cache: boolean that indates whether all anomaly scores should be computed at once or when asked for.
        :return:
        '''
        assert isinstance(visualisation_model, EventDataModel)
        assert isinstance(
            start_day, int) or isinstance(start_day, str) or isinstance(
                start_day, datetime.date) or (isinstance(start_day, tuple)
                                              and len(start_day) == 3)
        HasTraits.__init__(self)

        self.used_cache_size = 0  # must be initialized
        self._data = visualisation_model._event_data_object

        self.num_of_shown_days = num_of_shown_days  # Updates self.used_cache_size

        self._vis_model = visualisation_model
        self._anomaly_detector = visualisation_model._anomaly_detector
        self.anomaly_detection_threshold = decision_threshold

        dates = visualisation_model._event_data_object.dates_
        self._data_times = array([datetools.to_datetime(d) for d in dates])

        self.source_names = list(
            unique(visualisation_model._event_data_object.sources_))
        self._data_sources = array([
            self.source_names.index(source)
            for source in visualisation_model._event_data_object.sources_
        ])
        self._num_of_sources = len(unique(
            self.source_names))  # number of sources

        self.barcharts = []
        self.barchart_actors = []
        self.time_text3ds = []
        self.source_text3ds = []
        self.xy_positions = []

        self._high_start_day_number = int(
            (self._data_times.max() - self._data_times.min()).days)

        self.scene.anti_aliasing_frames = 8

        # add traits dynamically
        self.add_trait("Relative_Start_Day",
                       Range(0, self._high_start_day_number))

        self.add_trait("_selected_source_name",
                       Enum(None, [None] + self.source_names))

        self.configure_traits()

        self.scene.background = self.background

        # add the mouse pick handler
        self.picker = self.scene.mayavi_scene.on_mouse_pick(
            self.vis_picker, 'cell')

        self.picker.tolerance = 0.01

        #cmap = matplotlib.cm.get_cmap('Reds')
        #self.severity_color = [cmap(x)[:-1] for x in linspace(0.75, 0.95, self._vis_model.num_of_severity_levels_)]

        if self._vis_model.num_of_severity_levels_ > 1:
            self.severity_color = self.severity_color = [
                (1, x / 100.0, x / 100.0)
                for x in range(70, 30, -40 /
                               self._vis_model.num_of_severity_levels_)
            ]
        else:
            self.severity_color = [(255.0 / 255, 51 / 255.0, 51 / 255.0)]

        # This used for a fix to manage a bug in Mayavi library, an invisible default object
        self._obj = self.scene.mlab.points3d(0, 0, 0, opacity=0.0)

        # Cache all anomaly calculations for all data values
        if precompute_cache:
            self.used_cache_size = len(self._data)
            for data_index in xrange(len(self._data)):
                self._populate_cache(data_index)

        self.start_day = start_day

        self.update()