def _convert_listlike(arg, box):
if isinstance(arg, (list,tuple)):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError as e:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
return arg
arg = com._ensure_object(arg)
try:
if format is not None:
result = tslib.array_strptime(arg, format)
else:
result = tslib.array_to_datetime(arg, raise_=errors == 'raise',
utc=utc, dayfirst=dayfirst,
coerce=coerce, unit=unit)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def test_cached_range(self):
rng = DatetimeIndex._cached_range(START, END, offset=datetools.bday)
rng = DatetimeIndex._cached_range(START,
periods=20,
offset=datetools.bday)
rng = DatetimeIndex._cached_range(end=START,
periods=20,
offset=datetools.bday)
assertRaisesRegexp(TypeError, "offset", DatetimeIndex._cached_range,
START, END)
assertRaisesRegexp(TypeError,
"specify period",
DatetimeIndex._cached_range,
START,
offset=datetools.bday)
assertRaisesRegexp(TypeError,
"specify period",
DatetimeIndex._cached_range,
end=END,
offset=datetools.bday)
assertRaisesRegexp(TypeError,
"start or end",
DatetimeIndex._cached_range,
periods=20,
offset=datetools.bday)
def test_cdaterange_weekmask_and_holidays(self):
rng = cdate_range('2013-05-01',
periods=3,
weekmask='Sun Mon Tue Wed Thu',
holidays=['2013-05-01'])
xp = DatetimeIndex(['2013-05-02', '2013-05-05', '2013-05-06'])
self.assert_(xp.equals(rng))
def test_precision_finer_than_offset(self):
# GH 9907
result1 = DatetimeIndex(start='2015-04-15 00:00:03',
end='2016-04-22 00:00:00',
freq='Q')
result2 = DatetimeIndex(start='2015-04-15 00:00:03',
end='2015-06-22 00:00:04',
freq='W')
expected1_list = [
'2015-06-30 00:00:03', '2015-09-30 00:00:03',
'2015-12-31 00:00:03', '2016-03-31 00:00:03'
]
expected2_list = [
'2015-04-19 00:00:03', '2015-04-26 00:00:03',
'2015-05-03 00:00:03', '2015-05-10 00:00:03',
'2015-05-17 00:00:03', '2015-05-24 00:00:03',
'2015-05-31 00:00:03', '2015-06-07 00:00:03',
'2015-06-14 00:00:03', '2015-06-21 00:00:03'
]
expected1 = DatetimeIndex(expected1_list,
dtype='datetime64[ns]',
freq='Q-DEC',
tz=None)
expected2 = DatetimeIndex(expected2_list,
dtype='datetime64[ns]',
freq='W-SUN',
tz=None)
self.assert_index_equal(result1, expected1)
self.assert_index_equal(result2, expected2)
def _get_time_bins(self, axis):
assert (isinstance(axis, DatetimeIndex))
if len(axis) == 0:
binner = labels = DatetimeIndex(data=[], freq=self.freq)
return binner, [], labels
first, last = _get_range_edges(axis,
self.begin,
self.end,
self.freq,
closed=self.closed,
base=self.base)
binner = DatetimeIndex(freq=self.freq, start=first, end=last)
# a little hack
trimmed = False
if len(binner) > 2 and binner[-2] == axis[-1]:
binner = binner[:-1]
trimmed = True
# general version, knowing nothing about relative frequencies
bins = lib.generate_bins_dt64(axis.asi8, binner.asi8, self.closed)
if self.label == 'right':
labels = binner[1:]
elif not trimmed:
labels = binner[:-1]
else:
labels = binner
return binner, bins, labels
def maybe_to_datetimelike(data, copy=False):
"""
return a DelegatedClass of a Series that is datetimelike
(e.g. datetime64[ns],timedelta64[ns] dtype or a Series of Periods)
raise TypeError if this is not possible.
Parameters
----------
data : Series
copy : boolean, default False
copy the input data
Returns
-------
DelegatedClass
"""
from pandas import Series
if not isinstance(data, Series):
raise TypeError("cannot convert an object of type {0} to a datetimelike index".format(type(data)))
index = data.index
if issubclass(data.dtype.type, np.datetime64):
return DatetimeProperties(DatetimeIndex(data, copy=copy, freq='infer'), index)
elif issubclass(data.dtype.type, np.timedelta64):
return TimedeltaProperties(TimedeltaIndex(data, copy=copy, freq='infer'), index)
else:
if is_period_arraylike(data):
return PeriodProperties(PeriodIndex(data, copy=copy), index)
if is_datetime_arraylike(data):
return DatetimeProperties(DatetimeIndex(data, copy=copy, freq='infer'), index)
raise TypeError("cannot convert an object of type {0} to a datetimelike index".format(type(data)))
def test_cached_range(self):
rng = DatetimeIndex._cached_range(START, END, offset=datetools.bday)
rng = DatetimeIndex._cached_range(START,
periods=20,
offset=datetools.bday)
rng = DatetimeIndex._cached_range(end=START,
periods=20,
offset=datetools.bday)
self.assertRaises(Exception, DatetimeIndex._cached_range, START, END)
self.assertRaises(Exception,
DatetimeIndex._cached_range,
START,
freq=datetools.bday)
self.assertRaises(Exception,
DatetimeIndex._cached_range,
end=END,
freq=datetools.bday)
self.assertRaises(Exception,
DatetimeIndex._cached_range,
periods=20,
freq=datetools.bday)
def _get_time_bins(self, ax):
if not isinstance(ax, DatetimeIndex):
raise TypeError('axis must be a DatetimeIndex, but got '
'an instance of %r' % type(ax).__name__)
if len(ax) == 0:
binner = labels = DatetimeIndex(data=[],
freq=self.freq,
name=ax.name)
return binner, [], labels
first, last = _get_range_edges(ax,
self.freq,
closed=self.closed,
base=self.base)
tz = ax.tz
binner = labels = DatetimeIndex(freq=self.freq,
start=first.replace(tzinfo=None),
end=last.replace(tzinfo=None),
tz=tz,
name=ax.name)
# a little hack
trimmed = False
if (len(binner) > 2 and binner[-2] == ax.max()
and self.closed == 'right'):
binner = binner[:-1]
trimmed = True
ax_values = ax.asi8
binner, bin_edges = self._adjust_bin_edges(binner, ax_values)
# general version, knowing nothing about relative frequencies
bins = lib.generate_bins_dt64(ax_values, bin_edges, self.closed)
if self.closed == 'right':
labels = binner
if self.label == 'right':
labels = labels[1:]
elif not trimmed:
labels = labels[:-1]
else:
if self.label == 'right':
labels = labels[1:]
elif not trimmed:
labels = labels[:-1]
if (ax_values == tslib.iNaT).any():
binner = binner.insert(0, tslib.NaT)
labels = labels.insert(0, tslib.NaT)
# if we end up with more labels than bins
# adjust the labels
# GH4076
if len(bins) < len(labels):
labels = labels[:len(bins)]
return binner, bins, labels
def test_shift_months(self):
s = DatetimeIndex([Timestamp('2000-01-05 00:15:00'), Timestamp(
'2000-01-31 00:23:00'), Timestamp('2000-01-01'), Timestamp(
'2000-02-29'), Timestamp('2000-12-31')])
for years in [-1, 0, 1]:
for months in [-2, 0, 2]:
actual = DatetimeIndex(tslib.shift_months(s.asi8, years * 12 +
months))
expected = DatetimeIndex([x + offsets.DateOffset(
years=years, months=months) for x in s])
tm.assert_index_equal(actual, expected)
def test_cached_range(self):
rng = DatetimeIndex._cached_range(START, END, offset=datetools.cday)
rng = DatetimeIndex._cached_range(START, periods=20, offset=datetools.cday)
rng = DatetimeIndex._cached_range(end=START, periods=20, offset=datetools.cday)
self.assertRaises(Exception, DatetimeIndex._cached_range, START, END)
self.assertRaises(Exception, DatetimeIndex._cached_range, START, freq=datetools.cday)
self.assertRaises(Exception, DatetimeIndex._cached_range, end=END, freq=datetools.cday)
self.assertRaises(Exception, DatetimeIndex._cached_range, periods=20, freq=datetools.cday)
def _get_time_bins(self, axis):
assert (isinstance(axis, DatetimeIndex))
if len(axis) == 0:
binner = labels = DatetimeIndex(data=[], freq=self.freq)
return binner, [], labels
first, last = _get_range_edges(axis,
self.begin,
self.end,
self.freq,
closed=self.closed,
base=self.base)
binner = labels = DatetimeIndex(freq=self.freq, start=first, end=last)
# a little hack
trimmed = False
if len(binner) > 2 and binner[-2] == axis[-1]:
binner = binner[:-1]
trimmed = True
ax_values = axis.asi8
bin_edges = binner.asi8
# Some hacks for > daily data, see #1471, #1458
if self.freq != 'D' and is_superperiod(self.freq, 'D'):
day_nanos = _delta_to_nanoseconds(timedelta(1))
if self.closed == 'right':
bin_edges = bin_edges + day_nanos - 1
else:
bin_edges = bin_edges + day_nanos
# intraday values on last day
if bin_edges[-2] > ax_values[-1]:
bin_edges = bin_edges[:-1]
binner = binner[:-1]
# general version, knowing nothing about relative frequencies
bins = lib.generate_bins_dt64(ax_values, bin_edges, self.closed)
if self.closed == 'right':
labels = binner
if self.label == 'right':
labels = labels[1:]
elif not trimmed:
labels = labels[:-1]
else:
if self.label == 'right':
labels = labels[1:]
elif not trimmed:
labels = labels[:-1]
return binner, bins, labels
def test_freq_divides_end_in_nanos(self):
# GH 10885
result_1 = date_range('2005-01-12 10:00', '2005-01-12 16:00',
freq='345min')
result_2 = date_range('2005-01-13 10:00', '2005-01-13 16:00',
freq='345min')
expected_1 = DatetimeIndex(['2005-01-12 10:00:00', '2005-01-12 15:45:00'],
dtype='datetime64[ns]', freq='345T', tz=None)
expected_2 = DatetimeIndex(['2005-01-13 10:00:00', '2005-01-13 15:45:00'],
dtype='datetime64[ns]', freq='345T', tz=None)
self.assertTrue(result_1.equals(expected_1))
self.assertTrue(result_2.equals(expected_2))
def test_cached_range(self):
rng = DatetimeIndex._cached_range(START, END, offset=datetools.bday)
rng = DatetimeIndex._cached_range(START, periods=20, offset=datetools.bday)
rng = DatetimeIndex._cached_range(end=START, periods=20, offset=datetools.bday)
assertRaisesRegexp(TypeError, "offset", DatetimeIndex._cached_range, START, END)
assertRaisesRegexp(TypeError, "specify period", DatetimeIndex._cached_range, START, offset=datetools.bday)
assertRaisesRegexp(TypeError, "specify period", DatetimeIndex._cached_range, end=END, offset=datetools.bday)
assertRaisesRegexp(TypeError, "start or end", DatetimeIndex._cached_range, periods=20, offset=datetools.bday)
def maybe_to_datetimelike(data, copy=False):
"""
return a DelegatedClass of a Series that is datetimelike
(e.g. datetime64[ns],timedelta64[ns] dtype or a Series of Periods)
raise TypeError if this is not possible.
Parameters
----------
data : Series
copy : boolean, default False
copy the input data
Returns
-------
DelegatedClass
"""
from pandas import Series
if not isinstance(data, Series):
raise TypeError("cannot convert an object of type {0} to a "
"datetimelike index".format(type(data)))
index = data.index
name = data.name
orig = data if is_categorical_dtype(data) else None
if orig is not None:
data = orig.values.categories
if is_datetime64_dtype(data.dtype):
return DatetimeProperties(DatetimeIndex(data, copy=copy, freq='infer'),
index, name=name, orig=orig)
elif is_datetime64tz_dtype(data.dtype):
return DatetimeProperties(DatetimeIndex(data, copy=copy, freq='infer',
ambiguous='infer'),
index, data.name, orig=orig)
elif is_timedelta64_dtype(data.dtype):
return TimedeltaProperties(TimedeltaIndex(data, copy=copy,
freq='infer'), index,
name=name, orig=orig)
else:
if is_period_arraylike(data):
return PeriodProperties(PeriodIndex(data, copy=copy), index,
name=name, orig=orig)
if is_datetime_arraylike(data):
return DatetimeProperties(DatetimeIndex(data, copy=copy,
freq='infer'), index,
name=name, orig=orig)
raise TypeError("cannot convert an object of type {0} to a "
"datetimelike index".format(type(data)))
def _convert_listlike(arg, box):
if isinstance(arg, (list,tuple)):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError, e:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
return arg
def _convert_f(arg):
arg = com._ensure_object(arg)
try:
result = lib.array_to_datetime(arg, raise_=errors == 'raise',
utc=utc, dayfirst=dayfirst)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError, e:
try:
values, tz = lib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def test_datetimeindex(self):
from pandas.tseries.index import date_range, DatetimeIndex
rng = date_range('1/1/2000', periods=20)
encoded = ujson.encode(rng)
decoded = DatetimeIndex(np.array(ujson.decode(encoded)))
self.assert_(rng.equals(decoded))
ts = Series(np.random.randn(len(rng)), index=rng)
decoded = Series(ujson.decode(ujson.encode(ts)))
idx_values = decoded.index.values.astype(np.int64)
decoded.index = DatetimeIndex(idx_values)
tm.assert_series_equal(np.round(ts, 5), decoded)
def _convert_listlike(arg, box):
if isinstance(arg, (list, tuple)):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError as e:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
return arg
arg = com._ensure_object(arg)
try:
if format is not None:
result = None
# shortcut formatting here
if format == '%Y%m%d':
try:
result = _attempt_YYYYMMDD(arg)
except:
raise ValueError(
"cannot convert the input to '%Y%m%d' date format")
# fallback
if result is None:
result = tslib.array_strptime(arg, format)
else:
result = tslib.array_to_datetime(arg,
raise_=errors == 'raise',
utc=utc,
dayfirst=dayfirst,
coerce=coerce,
unit=unit)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def infer_freq(index, warn=True):
"""
Infer the most likely frequency given the input index. If the frequency is
uncertain, a warning will be printed
Parameters
----------
index : DatetimeIndex
warn : boolean, default True
Returns
-------
freq : string or None
None if no discernible frequency
"""
from pandas.tseries.index import DatetimeIndex
if not isinstance(index, DatetimeIndex):
from pandas.tseries.period import PeriodIndex
if isinstance(index, PeriodIndex):
raise ValueError("PeriodIndex given. Check the `freq` attribute "
"instead of using infer_freq.")
index = DatetimeIndex(index)
inferer = _FrequencyInferer(index, warn=warn)
return inferer.get_freq()
def test_setops_preserve_freq(self):
for tz in [None, 'Asia/Tokyo', 'US/Eastern']:
rng = date_range('1/1/2000', '1/1/2002', name='idx', tz=tz)
result = rng[:50].union(rng[50:100])
self.assertEqual(result.name, rng.name)
self.assertEqual(result.freq, rng.freq)
self.assertEqual(result.tz, rng.tz)
result = rng[:50].union(rng[30:100])
self.assertEqual(result.name, rng.name)
self.assertEqual(result.freq, rng.freq)
self.assertEqual(result.tz, rng.tz)
result = rng[:50].union(rng[60:100])
self.assertEqual(result.name, rng.name)
self.assertIsNone(result.freq)
self.assertEqual(result.tz, rng.tz)
result = rng[:50].intersection(rng[25:75])
self.assertEqual(result.name, rng.name)
self.assertEqual(result.freqstr, 'D')
self.assertEqual(result.tz, rng.tz)
nofreq = DatetimeIndex(list(rng[25:75]), name='other')
result = rng[:50].union(nofreq)
self.assertIsNone(result.name)
self.assertEqual(result.freq, rng.freq)
self.assertEqual(result.tz, rng.tz)
result = rng[:50].intersection(nofreq)
self.assertIsNone(result.name)
self.assertEqual(result.freq, rng.freq)
self.assertEqual(result.tz, rng.tz)
def to_timestamp(self, freq=None, how='start'):
"""
Cast to DatetimeIndex
Parameters
----------
freq : string or DateOffset, default 'D'
Target frequency
how : {'s', 'e', 'start', 'end'}
Returns
-------
DatetimeIndex
"""
if freq is None:
base, mult = _gfc(self.freq)
new_data = self
else:
base, mult = _gfc(freq)
new_data = self.asfreq(freq, how)
if mult != 1:
raise ValueError('Only mult == 1 supported')
new_data = plib.periodarr_to_dt64arr(new_data.values, base)
return DatetimeIndex(new_data, freq='infer', name=self.name)
def __new__(cls,
start=None,
end=None,
periods=None,
offset=datetools.bday,
time_rule=None,
tzinfo=None,
name=None,
**kwds):
import warnings
warnings.warn("DateRange is deprecated, use DatetimeIndex instead",
FutureWarning)
if time_rule is None:
time_rule = kwds.get('timeRule')
if time_rule is not None:
offset = datetools.get_offset(time_rule)
return DatetimeIndex(start=start,
end=end,
periods=periods,
freq=offset,
tzinfo=tzinfo,
name=name,
**kwds)
def test_to_period(self):
from pandas.tseries.period import period_range
ts = _simple_ts('1/1/2000', '1/1/2001')
pts = ts.to_period()
exp = ts.copy()
exp.index = period_range('1/1/2000', '1/1/2001')
assert_series_equal(pts, exp)
pts = ts.to_period('M')
exp.index = exp.index.asfreq('M')
tm.assert_index_equal(pts.index, exp.index.asfreq('M'))
assert_series_equal(pts, exp)
# GH 7606 without freq
idx = DatetimeIndex(['2011-01-01', '2011-01-02', '2011-01-03',
'2011-01-04'])
exp_idx = pd.PeriodIndex(['2011-01-01', '2011-01-02', '2011-01-03',
'2011-01-04'], freq='D')
s = Series(np.random.randn(4), index=idx)
expected = s.copy()
expected.index = exp_idx
assert_series_equal(s.to_period(), expected)
df = DataFrame(np.random.randn(4, 4), index=idx, columns=idx)
expected = df.copy()
expected.index = exp_idx
assert_frame_equal(df.to_period(), expected)
expected = df.copy()
expected.columns = exp_idx
assert_frame_equal(df.to_period(axis=1), expected)
def _concat_compat(to_concat, axis=0):
"""
provide concatenation of an datetimelike array of arrays each of which is a single
M8[ns], datetimet64[ns, tz] or m8[ns] dtype
Parameters
----------
to_concat : array of arrays
axis : axis to provide concatenation
Returns
-------
a single array, preserving the combined dtypes
"""
def convert_to_pydatetime(x, axis):
# coerce to an object dtype
if x.dtype == _NS_DTYPE:
if hasattr(x, 'tz'):
x = x.asobject
shape = x.shape
x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
x = x.reshape(shape)
elif x.dtype == _TD_DTYPE:
shape = x.shape
x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
x = x.reshape(shape)
return x
typs = get_dtype_kinds(to_concat)
# datetimetz
if 'datetimetz' in typs:
# we require ALL of the same tz for datetimetz
tzs = set([getattr(x, 'tz', None) for x in to_concat]) - set([None])
if len(tzs) == 1:
return DatetimeIndex(np.concatenate(
[x.tz_localize(None).asi8 for x in to_concat]),
tz=list(tzs)[0])
# single dtype
if len(typs) == 1:
if not len(typs - set(['datetime'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_NS_DTYPE)
elif not len(typs - set(['timedelta'])):
new_values = np.concatenate([x.view(np.int64) for x in to_concat],
axis=axis)
return new_values.view(_TD_DTYPE)
# need to coerce to object
to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
return np.concatenate(to_concat, axis=axis)
def main():
nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb"
sim_name = "crcm5-r"
sim_folder = os.path.join(nc_db_folder, "{0}".format(sim_name))
var_name = "TT"
nc_path = os.path.join(sim_folder, "{0}_all.nc4".format(var_name))
ds = Dataset(nc_path)
timeVar = ds.variables["time"]
print(timeVar.units)
t0 = num2date(timeVar[0], timeVar.units)
t1 = num2date(timeVar[1], timeVar.units)
tf = num2date(timeVar[-1], timeVar.units)
dt = t1 - t0
dr = DatetimeIndex(start=t0, end=tf, freq=DateOffset(seconds=dt.seconds))
#print len( dr.tolist() )
print(t0, t1, tf)
#print help( h.root.TT )
pass
def to_timestamp(self, freq=None, how='start'):
"""
Cast to DatetimeIndex
Parameters
----------
freq : string or DateOffset, default 'D' for week or longer, 'S'
otherwise
Target frequency
how : {'s', 'e', 'start', 'end'}
Returns
-------
DatetimeIndex
"""
how = _validate_end_alias(how)
if freq is None:
base, mult = _gfc(self.freq)
freq = frequencies.get_to_timestamp_base(base)
else:
freq = Period._maybe_convert_freq(freq)
base, mult = _gfc(freq)
new_data = self.asfreq(freq, how)
new_data = period.periodarr_to_dt64arr(new_data._values, base)
return DatetimeIndex(new_data, freq='infer', name=self.name)
def _get_time_bins(self, axis):
if not isinstance(axis, DatetimeIndex):
raise TypeError('axis must be a DatetimeIndex, but got '
'an instance of %r' % type(axis).__name__)
if len(axis) == 0:
binner = labels = DatetimeIndex(data=[], freq=self.freq)
return binner, [], labels
first, last = _get_range_edges(axis,
self.freq,
closed=self.closed,
base=self.base)
tz = axis.tz
binner = labels = DatetimeIndex(freq=self.freq,
start=first.replace(tzinfo=None),
end=last.replace(tzinfo=None),
tz=tz)
# a little hack
trimmed = False
if (len(binner) > 2 and binner[-2] == axis[-1]
and self.closed == 'right'):
binner = binner[:-1]
trimmed = True
ax_values = axis.asi8
binner, bin_edges = self._adjust_bin_edges(binner, ax_values)
# general version, knowing nothing about relative frequencies
bins = lib.generate_bins_dt64(ax_values, bin_edges, self.closed)
if self.closed == 'right':
labels = binner
if self.label == 'right':
labels = labels[1:]
elif not trimmed:
labels = labels[:-1]
else:
if self.label == 'right':
labels = labels[1:]
elif not trimmed:
labels = labels[:-1]
return binner, bins, labels
def test_range_bug(self):
# GH #770
offset = datetools.DateOffset(months=3)
result = date_range("2011-1-1", "2012-1-31", freq=offset)
start = datetime(2011, 1, 1)
exp_values = [start + i * offset for i in range(5)]
self.assertTrue(np.array_equal(result, DatetimeIndex(exp_values)))
def get_time_index(freq, start=None, end=None):
if start is None:
start = "1/1/2012 9:30AM"
if end is None:
end = "1/1/2012 4:00PM"
ideal = DatetimeIndex(start=start, end=end, freq=freq)
times = [date.time() for date in ideal]
return TimeIndex(times)
def get_anchor_index(index, freq):
ideal = get_time_index(freq)
start = index[0]
start = ideal.asof(start)
end = index[-1]
start, end = _get_range_edges(index, offset=freq, closed='right')
ind = DatetimeIndex(start=start, end=end, freq=freq)
return ind
def _convert_listlike(arg, box):
if isinstance(arg, (list,tuple)):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError as e:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
return arg
arg = com._ensure_object(arg)
try:
if format is not None:
result = None
# shortcut formatting here
if format == '%Y%m%d':
try:
result = _attempt_YYYYMMDD(arg)
except:
raise ValueError("cannot convert the input to '%Y%m%d' date format")
# fallback
if result is None:
result = tslib.array_strptime(arg, format)
else:
result = tslib.array_to_datetime(arg, raise_=errors == 'raise',
utc=utc, dayfirst=dayfirst,
coerce=coerce, unit=unit)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def test_precision_finer_than_offset(self):
# GH 9907
result1 = DatetimeIndex(start='2015-04-15 00:00:03',
end='2016-04-22 00:00:00', freq='Q')
result2 = DatetimeIndex(start='2015-04-15 00:00:03',
end='2015-06-22 00:00:04', freq='W')
expected1_list = ['2015-06-30 00:00:03', '2015-09-30 00:00:03',
'2015-12-31 00:00:03', '2016-03-31 00:00:03']
expected2_list = ['2015-04-19 00:00:03', '2015-04-26 00:00:03',
'2015-05-03 00:00:03', '2015-05-10 00:00:03',
'2015-05-17 00:00:03', '2015-05-24 00:00:03',
'2015-05-31 00:00:03', '2015-06-07 00:00:03',
'2015-06-14 00:00:03', '2015-06-21 00:00:03']
expected1 = DatetimeIndex(expected1_list, dtype='datetime64[ns]',
freq='Q-DEC', tz=None)
expected2 = DatetimeIndex(expected2_list, dtype='datetime64[ns]',
freq='W-SUN', tz=None)
self.assertTrue(result1.equals(expected1))
self.assertTrue(result2.equals(expected2))
def test_read_all_cols_all_dtypes(tickstore_lib, chunk_size):
data = [
{
'f': 0.1,
'of': 0.2,
's': 's',
'os': 'os',
'l': 1,
'ol': 2,
'index': dt(1970, 1, 1, tzinfo=mktz('UTC')),
},
{
'f': 0.3,
'nf': 0.4,
's': 't',
'ns': 'ns',
'l': 3,
'nl': 4,
'index': dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC')),
},
]
tickstore_lib.chunk_size = 3
tickstore_lib.write('sym', data)
df = tickstore_lib.read('sym', columns=None)
# The below is probably more trouble than it's worth, but we *should*
# be able to roundtrip data and get the same answer...
# Ints become floats
data[0]['l'] = float(data[0]['l'])
# Treat missing strings as None
data[0]['ns'] = None
data[1]['os'] = None
index = DatetimeIndex([
dt(1970, 1, 1, tzinfo=mktz('UTC')),
dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC'))
], )
index.tz = mktz()
expected = pd.DataFrame(data, index=index)
expected = expected[df.columns]
assert_frame_equal(expected, df, check_names=False)
def inferTimeRule(index):
from pandas.tseries.index import DatetimeIndex
import warnings
warnings.warn("This method is deprecated, use infer_freq or inferred_freq"
" attribute of DatetimeIndex", FutureWarning)
freq = DatetimeIndex(index).inferred_freq
if freq is None:
raise Exception('Unable to infer time rule')
offset = to_offset(freq)
return get_legacy_offset_name(offset)
def test_read_all_cols_all_dtypes(tickstore_lib, chunk_size):
data = [{'f': 0.1,
'of': 0.2,
's': 's',
'os': 'os',
'l': 1,
'ol': 2,
'index': dt(1970, 1, 1, tzinfo=mktz('UTC')),
},
{'f': 0.3,
'nf': 0.4,
's': 't',
'ns': 'ns',
'l': 3,
'nl': 4,
'index': dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC')),
},
]
tickstore_lib._chunk_size = chunk_size
tickstore_lib.write('sym', data)
df = tickstore_lib.read('sym', columns=None)
# The below is probably more trouble than it's worth, but we *should*
# be able to roundtrip data and get the same answer...
# Ints become floats
data[0]['l'] = float(data[0]['l'])
# Treat missing strings as None
data[0]['ns'] = None
data[1]['os'] = None
index = DatetimeIndex([dt(1970, 1, 1, tzinfo=mktz('UTC')),
dt(1970, 1, 1, 0, 0, 1, tzinfo=mktz('UTC'))],
)
index.tz = mktz()
expected = pd.DataFrame(data, index=index)
expected = expected[df.columns]
assert_frame_equal(expected, df, check_names=False)
def _convert_listlike(arg, box):
if isinstance(arg, (list, tuple)):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError as e:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
return arg
arg = com._ensure_object(arg)
try:
if format is not None:
result = tslib.array_strptime(arg, format)
else:
result = tslib.array_to_datetime(arg,
raise_=errors == 'raise',
utc=utc,
dayfirst=dayfirst,
coerce=coerce,
unit=unit)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def _convert_listlike(arg):
if isinstance(arg, list):
arg = np.array(arg, dtype="O")
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz="utc" if utc else None)
except ValueError, e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
return arg
def _convert_f(arg):
arg = com._ensure_object(arg)
try:
result = lib.array_to_datetime(arg, raise_=errors == 'raise',
utc=utc, dayfirst=dayfirst)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError, e:
try:
values, tz = lib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def _convert_f(arg):
arg = com._ensure_object(arg)
try:
if format is not None:
result = tslib.array_strptime(arg, format)
else:
result = tslib.array_to_datetime(
arg, raise_=errors == "raise", utc=utc, dayfirst=dayfirst, coerce=coerce, unit=unit
)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz="utc" if utc else None)
return result
except ValueError, e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def test_cached_range(self):
DatetimeIndex._cached_range(START, END, offset=CDay())
DatetimeIndex._cached_range(START, periods=20,
offset=CDay())
DatetimeIndex._cached_range(end=START, periods=20,
offset=CDay())
self.assertRaises(Exception, DatetimeIndex._cached_range, START, END)
self.assertRaises(Exception, DatetimeIndex._cached_range, START,
freq=CDay())
self.assertRaises(Exception, DatetimeIndex._cached_range, end=END,
freq=CDay())
self.assertRaises(Exception, DatetimeIndex._cached_range, periods=20,
freq=CDay())
def test_cdaterange_weekmask_and_holidays(self):
rng = cdate_range('2013-05-01', periods=3,
weekmask='Sun Mon Tue Wed Thu',
holidays=['2013-05-01'])
xp = DatetimeIndex(['2013-05-02', '2013-05-05', '2013-05-06'])
self.assertTrue(xp.equals(rng))
def test_cdaterange_holidays(self):
rng = cdate_range('2013-05-01', periods=3,
holidays=['2013-05-01'])
xp = DatetimeIndex(['2013-05-02', '2013-05-03', '2013-05-06'])
self.assertTrue(xp.equals(rng))
def test_cdaterange(self):
rng = cdate_range('2013-05-01', periods=3)
xp = DatetimeIndex(['2013-05-01', '2013-05-02', '2013-05-03'])
self.assertTrue(xp.equals(rng))
def _convert_listlike(arg, box, format):
if isinstance(arg, (list, tuple)):
arg = np.array(arg, dtype="O")
if com.is_datetime64_ns_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz="utc" if utc else None)
except ValueError:
pass
return arg
arg = com._ensure_object(arg)
if infer_datetime_format and format is None:
format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
if format is not None:
# There is a special fast-path for iso8601 formatted
# datetime strings, so in those cases don't use the inferred
# format because this path makes process slower in this
# special case
format_is_iso8601 = "%Y-%m-%dT%H:%M:%S.%f".startswith(format) or "%Y-%m-%d %H:%M:%S.%f".startswith(
format
)
if format_is_iso8601:
format = None
try:
result = None
if format is not None:
# shortcut formatting here
if format == "%Y%m%d":
try:
result = _attempt_YYYYMMDD(arg)
except:
raise ValueError("cannot convert the input to '%Y%m%d' date format")
# fallback
if result is None:
try:
result = tslib.array_strptime(arg, format, coerce=coerce)
except (tslib.OutOfBoundsDatetime):
if errors == "raise":
raise
result = arg
except ValueError:
# Only raise this error if the user provided the
# datetime format, and not when it was inferred
if not infer_datetime_format:
raise
if result is None and (format is None or infer_datetime_format):
result = tslib.array_to_datetime(
arg, raise_=errors == "raise", utc=utc, dayfirst=dayfirst, coerce=coerce, unit=unit
)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz="utc" if utc else None)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def test_cdaterange(self):
rng = cdate_range("2013-05-01", periods=3)
xp = DatetimeIndex(["2013-05-01", "2013-05-02", "2013-05-03"])
self.assert_(xp.equals(rng))
def _convert_listlike(arg, box, format, name=None, tz=tz):
if isinstance(arg, (list, tuple)):
arg = np.array(arg, dtype='O')
# these are shortcutable
if is_datetime64_ns_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz=tz, name=name)
except ValueError:
pass
return arg
elif is_datetime64tz_dtype(arg):
if not isinstance(arg, DatetimeIndex):
return DatetimeIndex(arg, tz=tz, name=name)
if utc:
arg = arg.tz_convert(None).tz_localize('UTC')
return arg
elif unit is not None:
if format is not None:
raise ValueError("cannot specify both format and unit")
arg = getattr(arg, 'values', arg)
result = tslib.array_with_unit_to_datetime(arg, unit,
errors=errors)
if box:
if errors == 'ignore':
from pandas import Index
return Index(result)
return DatetimeIndex(result, tz=tz, name=name)
return result
elif getattr(arg, 'ndim', 1) > 1:
raise TypeError('arg must be a string, datetime, list, tuple, '
'1-d array, or Series')
arg = _ensure_object(arg)
require_iso8601 = False
if infer_datetime_format and format is None:
format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
if format is not None:
# There is a special fast-path for iso8601 formatted
# datetime strings, so in those cases don't use the inferred
# format because this path makes process slower in this
# special case
format_is_iso8601 = _format_is_iso(format)
if format_is_iso8601:
require_iso8601 = not infer_datetime_format
format = None
try:
result = None
if format is not None:
# shortcut formatting here
if format == '%Y%m%d':
try:
result = _attempt_YYYYMMDD(arg, errors=errors)
except:
raise ValueError("cannot convert the input to "
"'%Y%m%d' date format")
# fallback
if result is None:
try:
result = tslib.array_strptime(arg, format, exact=exact,
errors=errors)
except tslib.OutOfBoundsDatetime:
if errors == 'raise':
raise
result = arg
except ValueError:
# if format was inferred, try falling back
# to array_to_datetime - terminate here
# for specified formats
if not infer_datetime_format:
if errors == 'raise':
raise
result = arg
if result is None and (format is None or infer_datetime_format):
result = tslib.array_to_datetime(
arg,
errors=errors,
utc=utc,
dayfirst=dayfirst,
yearfirst=yearfirst,
require_iso8601=require_iso8601
)
if is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz=tz, name=name)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, name=name, tz=tz)
except (ValueError, TypeError):
raise e
def to_datetime(arg, errors='ignore', dayfirst=False, utc=None, box=True,
format=None, coerce=False, unit='ns'):
"""
Convert argument to datetime
Parameters
----------
arg : string, datetime, array of strings (with possible NAs)
errors : {'ignore', 'raise'}, default 'ignore'
Errors are ignored by default (values left untouched)
dayfirst : boolean, default False
If True parses dates with the day first, eg 20/01/2005
Warning: dayfirst=True is not strict, but will prefer to parse
with day first (this is a known bug).
utc : boolean, default None
Return UTC DatetimeIndex if True (converting any tz-aware
datetime.datetime objects as well)
box : boolean, default True
If True returns a DatetimeIndex, if False returns ndarray of values
format : string, default None
strftime to parse time, eg "%d/%m/%Y"
coerce : force errors to NaT (False by default)
unit : unit of the arg (D,s,ms,us,ns) denote the unit in epoch
(e.g. a unix timestamp), which is an integer/float number
Returns
-------
ret : datetime if parsing succeeded
"""
from pandas import Timestamp
from pandas.core.series import Series
from pandas.tseries.index import DatetimeIndex
def _convert_listlike(arg, box):
if isinstance(arg, (list,tuple)):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError, e:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
return arg
arg = com._ensure_object(arg)
try:
if format is not None:
result = tslib.array_strptime(arg, format)
else:
result = tslib.array_to_datetime(arg, raise_=errors == 'raise',
utc=utc, dayfirst=dayfirst,
coerce=coerce, unit=unit)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError, e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
def test_cdaterange_weekmask(self):
rng = cdate_range('2013-05-01', periods=3,
weekmask='Sun Mon Tue Wed Thu')
xp = DatetimeIndex(['2013-05-01', '2013-05-02', '2013-05-05'])
self.assert_(xp.equals(rng))
def test_cdaterange_holidays(self):
rng = cdate_range("2013-05-01", periods=3, holidays=["2013-05-01"])
xp = DatetimeIndex(["2013-05-02", "2013-05-03", "2013-05-06"])
self.assert_(xp.equals(rng))
def _convert_listlike(arg, box, format):
if isinstance(arg, (list,tuple)):
arg = np.array(arg, dtype='O')
# these are shortcutable
if com.is_datetime64_ns_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError:
pass
return arg
elif format is None and com.is_integer_dtype(arg) and unit=='ns':
result = arg.astype('datetime64[ns]')
if box:
return DatetimeIndex(result, tz='utc' if utc else None)
return result
arg = com._ensure_object(arg)
require_iso8601 = False
if infer_datetime_format and format is None:
format = _guess_datetime_format_for_array(arg, dayfirst=dayfirst)
if format is not None:
# There is a special fast-path for iso8601 formatted
# datetime strings, so in those cases don't use the inferred
# format because this path makes process slower in this
# special case
format_is_iso8601 = (
('%Y-%m-%dT%H:%M:%S.%f'.startswith(format) or
'%Y-%m-%d %H:%M:%S.%f'.startswith(format)) and
format != '%Y'
)
if format_is_iso8601:
require_iso8601 = not infer_datetime_format
format = None
try:
result = None
if format is not None:
# shortcut formatting here
if format == '%Y%m%d':
try:
result = _attempt_YYYYMMDD(arg, coerce=coerce)
except:
raise ValueError("cannot convert the input to '%Y%m%d' date format")
# fallback
if result is None:
try:
result = tslib.array_strptime(
arg, format, exact=exact, coerce=coerce
)
except (tslib.OutOfBoundsDatetime):
if errors == 'raise':
raise
result = arg
except ValueError:
# if format was inferred, try falling back
# to array_to_datetime - terminate here
# for specified formats
if not infer_datetime_format:
if errors == 'raise':
raise
result = arg
if result is None and (format is None or infer_datetime_format):
result = tslib.array_to_datetime(arg, raise_=errors=='raise',
utc=utc, dayfirst=dayfirst,
yearfirst=yearfirst, freq=freq,
coerce=coerce, unit=unit,
require_iso8601=require_iso8601)
if com.is_datetime64_dtype(result) and box:
result = DatetimeIndex(result, tz='utc' if utc else None)
return result
except ValueError as e:
try:
values, tz = tslib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
return arg
elif isinstance(arg, Series):
values = _convert_f(arg.values)
return Series(values, index=arg.index, name=arg.name)
elif isinstance(arg, (np.ndarray, list)):
if isinstance(arg, list):
arg = np.array(arg, dtype='O')
if com.is_datetime64_dtype(arg):
if box and not isinstance(arg, DatetimeIndex):
try:
return DatetimeIndex(arg, tz='utc' if utc else None)
except ValueError, e:
try:
values, tz = lib.datetime_to_datetime64(arg)
return DatetimeIndex._simple_new(values, None, tz=tz)
except (ValueError, TypeError):
raise e
return arg
try:
return _convert_f(arg)
except ValueError:
raise
return arg
try:
if not arg:
return arg
return parse(arg, dayfirst=dayfirst)
except Exception:
def test_cdaterange_weekmask_and_holidays(self):
rng = cdate_range("2013-05-01", periods=3, weekmask="Sun Mon Tue Wed Thu", holidays=["2013-05-01"])
xp = DatetimeIndex(["2013-05-02", "2013-05-05", "2013-05-06"])
self.assert_(xp.equals(rng))
