def test_replace_integer_args(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
ts = Timestamp("2016-01-01 09:00:00.000000123", tz=tz)
msg = "value must be an integer, received <class 'float'> for hour"
with pytest.raises(ValueError, match=msg):
ts.replace(hour=0.1)
class TimestampOps(object):
params = [None, 'US/Eastern', pytz.UTC, dateutil.tz.tzutc()]
param_names = ['tz']
def setup(self, tz):
self.ts = Timestamp('2017-08-25 08:16:14', tz=tz)
def time_replace_tz(self, tz):
self.ts.replace(tzinfo=pytz.timezone('US/Eastern'))
def time_replace_None(self, tz):
self.ts.replace(tzinfo=None)
def time_to_pydatetime(self, tz):
self.ts.to_pydatetime()
def time_normalize(self, tz):
self.ts.normalize()
def time_tz_convert(self, tz):
if self.ts.tz is not None:
self.ts.tz_convert(tz)
def time_tz_localize(self, tz):
if self.ts.tz is None:
self.ts.tz_localize(tz)
def time_to_julian_date(self, tz):
self.ts.to_julian_date()
def time_floor(self, tz):
self.ts.floor('5T')
def time_ceil(self, tz):
self.ts.ceil('5T')
def test_replace_invalid_kwarg(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
ts = Timestamp("2016-01-01 09:00:00.000000123", tz=tz)
msg = r"replace\(\) got an unexpected keyword argument"
with pytest.raises(TypeError, match=msg):
ts.replace(foo=5)
class TimestampOps(object):
params = [None, 'US/Eastern', pytz.UTC,
dateutil.tz.tzutc()]
param_names = ['tz']
def setup(self, tz):
self.ts = Timestamp('2017-08-25 08:16:14', tz=tz)
def time_replace_tz(self, tz):
self.ts.replace(tzinfo=pytz.timezone('US/Eastern'))
def time_replace_None(self, tz):
self.ts.replace(tzinfo=None)
def time_to_pydatetime(self, tz):
self.ts.to_pydatetime()
def time_normalize(self, tz):
self.ts.normalize()
def time_tz_convert(self, tz):
if self.ts.tz is not None:
self.ts.tz_convert(tz)
def time_tz_localize(self, tz):
if self.ts.tz is None:
self.ts.tz_localize(tz)
class TimestampOps:
params = [None, "US/Eastern", pytz.UTC, dateutil.tz.tzutc()]
param_names = ["tz"]
def setup(self, tz):
self.ts = Timestamp("2017-08-25 08:16:14", tz=tz)
def time_replace_tz(self, tz):
self.ts.replace(tzinfo=pytz.timezone("US/Eastern"))
def time_replace_None(self, tz):
self.ts.replace(tzinfo=None)
def time_to_pydatetime(self, tz):
self.ts.to_pydatetime()
def time_normalize(self, tz):
self.ts.normalize()
def time_tz_convert(self, tz):
if self.ts.tz is not None:
self.ts.tz_convert(tz)
def time_tz_localize(self, tz):
if self.ts.tz is None:
self.ts.tz_localize(tz)
def time_to_julian_date(self, tz):
self.ts.to_julian_date()
def time_floor(self, tz):
self.ts.floor("5T")
def time_ceil(self, tz):
self.ts.ceil("5T")
class TimestampAcrossDst:
def setup(self):
dt = datetime.datetime(2016, 3, 27, 1)
self.tzinfo = pytz.timezone("CET").localize(dt, is_dst=False).tzinfo
self.ts2 = Timestamp(dt)
def time_replace_across_dst(self):
self.ts2.replace(tzinfo=self.tzinfo)
class TimestampAcrossDst:
def setup(self):
dt = datetime.datetime(2016, 3, 27, 1)
self.tzinfo = pytz.timezone('CET').localize(dt, is_dst=False).tzinfo
self.ts2 = Timestamp(dt)
def time_replace_across_dst(self):
self.ts2.replace(tzinfo=self.tzinfo)
def _convert_event_timestamp(event_timestamp: pd.Timestamp, t: EventTimestampType):
if t == EventTimestampType.TZ_NAIVE:
return event_timestamp
elif t == EventTimestampType.TZ_AWARE_UTC:
return event_timestamp.replace(tzinfo=utc)
elif t == EventTimestampType.TZ_AWARE_FIXED_OFFSET:
return event_timestamp.replace(tzinfo=utc).astimezone(FixedOffset(60))
elif t == EventTimestampType.TZ_AWARE_US_PACIFIC:
return event_timestamp.replace(tzinfo=utc).astimezone(timezone("US/Pacific"))
def subtract_years(dt: pd.Timestamp, years: int) -> pd.Timestamp:
"""
Subtracts N years (integer) from a date. Used for time series.
First month is +1 (if today is August the series should start at September to give 12 months).
"""
if isinstance(years, int):
if dt.month == 12:
dt = dt.replace(year=dt.year - years, month=1) # for December
else:
dt = dt.replace(year=dt.year - years, month=dt.month + 1)
else:
raise TypeError('The period should be integer')
return dt
class TimestampOps(object):
params = [None, 'US/Eastern']
param_names = ['tz']
def setup(self, tz):
self.ts = Timestamp('2017-08-25 08:16:14', tz=tz)
def time_replace_tz(self, tz):
self.ts.replace(tzinfo=pytz.timezone('US/Eastern'))
def time_replace_None(self, tz):
self.ts.replace(tzinfo=None)
def time_to_pydatetime(self, tz):
self.ts.to_pydatetime()
def test_replace_preserves_nanos(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
result = ts.replace(hour=0)
expected = Timestamp('2016-01-01 00:00:00.000000123', tz=tz)
assert result == expected
def generate_data_file(users: pd.DataFrame, timestamp: pd.Timestamp, filter_items: list, t: tqdm):
sensor_data = pd.DataFrame(
columns=['userid', 'timestamp', 'sensor01', 'sensor02', 'sensor03', 'sensor04'])
user_readings = {user['userid']: [generate_sensor_readings(
user['type'], 24) for _ in range(4)] for _, user in users.iterrows()}
for hour in range(0, 24):
for _, user in users.iterrows():
entry = {
'userid': user['userid'],
'timestamp': timestamp.replace(hour=hour),
'sensor01': user_readings[user['userid']][0][hour],
'sensor02': user_readings[user['userid']][1][hour],
'sensor03': user_readings[user['userid']][2][hour],
'sensor04': user_readings[user['userid']][3][hour]
}
sensor_data = sensor_data.append(entry, ignore_index=True)
t.update()
sensor_data = sensor_data.filter(
items=['userid', 'timestamp']+filter_items)
year = "{:04d}".format(timestamp.year)
month = "{:02d}".format(timestamp.month)
outdir = root_path(f'data/{year}/{month}/')
filename = f"data_{str(timestamp).split(' ')[0]}.csv"
sensor_data.to_csv(ensure_path(outdir, filename), index=False)
return sensor_data
def test_replace_aware(self, tz):
# GH#14621, GH#7825
# replacing datetime components with and w/o presence of a timezone
ts = Timestamp('2016-01-01 09:00:00', tz=tz)
result = ts.replace(hour=0)
expected = Timestamp('2016-01-01 00:00:00', tz=tz)
assert result == expected
def test_replace_preserves_nanos(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
ts = Timestamp("2016-01-01 09:00:00.000000123", tz=tz)
result = ts.replace(hour=0)
expected = Timestamp("2016-01-01 00:00:00.000000123", tz=tz)
assert result == expected
def test_replace_aware(self, tz):
# GH#14621, GH#7825
# replacing datetime components with and w/o presence of a timezone
ts = Timestamp('2016-01-01 09:00:00', tz=tz)
result = ts.replace(hour=0)
expected = Timestamp('2016-01-01 00:00:00', tz=tz)
assert result == expected
def test_replace_dst_fold(self, fold, tz):
# GH 25017
d = datetime(2019, 10, 27, 2, 30)
ts = Timestamp(d, tz=tz)
result = ts.replace(hour=1, fold=fold)
expected = Timestamp(datetime(2019, 10, 27, 1,
30)).tz_localize(tz, ambiguous=not fold)
assert result == expected
def _produce_perf_data(seed: int, start_date: Timestamp,
end_date: Timestamp, max: float, trend_adj: float,
amt: float) -> DataFrame:
"""
Produce randomised performance data based on the provided parameters
:param int seed: The seed to use to initialise the random number generator
:param Timestamp start_date: The start date of the performance
:param Timestamp end_date: The end date of the performance
:param float max: The maximum daily return possible
:param float trend_adj: An adjustment to make on each daily return
:param float amt: The starting market value
:return: DataFrame df: The DataFrame containing the random performance data
"""
start_date = start_date.replace(tzinfo=pytz.UTC)
end_date = end_date.replace(tzinfo=pytz.UTC)
# Create a shell DataFrame
df = pd.DataFrame(data=pd.date_range(start_date, end_date),
columns=['date'])
# Get the number of days in the DataFrame
num = len(df)
# Seed the random number generator
rnd.seed(seed)
'''
1) Create a series of random returns with size of the DataFrame + 4
2) Smooth out the returns by taking the rolling 5 day mean
3) Shift back 3 places to fill in missing data where window was less than 5 days, leaving the first day NaN
4) Ensure that the series is the same length as the DataFrame
5) Fill the missing first day with 1
6) Turn the series into a cumulative product and multiple this by the initial market value to get a series
of market values
7) Round to 2 decimal places
'''
df['mv'] = np.round(
pd.Series(data=(rnd.random(num + 4) * max * 2.0) + trend_adj - max)
.rolling(5).mean().shift(-3).head(num).fillna(1.0).cumprod() * amt,
2)
# No flows
df['net'] = 0.0
df['key'] = 'all'
return df
def test_replace_multiple(self, tz):
# GH#14621, GH#7825
# replacing datetime components with and w/o presence of a timezone
# test all
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
result = ts.replace(year=2015, month=2, day=2, hour=0, minute=5,
second=5, microsecond=5, nanosecond=5)
expected = Timestamp('2015-02-02 00:05:05.000005005', tz=tz)
assert result == expected
async def do_strategies(
trader: Trader,
data_loader: DataLoader,
symbol: str,
position: float,
now: pd.Timestamp,
data: Dict,
) -> None:
# run strategies
for s in trading_data.strategies:
if ("symbol_strategy" in data and data["symbol_strategy"]
and s.name != data["symbol_strategy"]):
continue
if s.name in rejects and symbol in rejects[s.name]:
continue
try:
skip = await s.should_run_all()
except Exception:
skip = False
finally:
if skip:
continue
try:
do, what = await s.run(
symbol=symbol,
shortable=shortable[symbol],
position=position,
minute_history=data_loader[symbol].symbol_data,
now=pd.to_datetime(now.replace(nanosecond=0)).replace(
second=0, microsecond=0),
portfolio_value=config.portfolio_value,
)
except Exception:
traceback.print_exc()
exc_info = sys.exc_info()
lines = traceback.format_exception(*exc_info)
for line in lines:
tlog(f"{line}")
del exc_info
if do:
await execute_strategy_result(
strategy=s,
trader=trader,
data_loader=data_loader,
symbol=symbol,
what=what,
)
else:
if what.get("reject", False):
if s.name not in rejects:
rejects[s.name] = [symbol]
else:
rejects[s.name].append(symbol)
def test_replace_dst_fold(self, fold, tz):
# GH 25017
d = datetime(2019, 10, 27, 2, 30)
ts = Timestamp(d, tz=tz)
result = ts.replace(hour=1, fold=fold)
expected = Timestamp(datetime(2019, 10, 27, 1, 30)).tz_localize(
tz, ambiguous=not fold
)
assert result == expected
def test_replace_multiple(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
# replacing datetime components with and w/o presence of a timezone
# test all
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
result = ts.replace(year=2015, month=2, day=2, hour=0, minute=5,
second=5, microsecond=5, nanosecond=5)
expected = Timestamp('2015-02-02 00:05:05.000005005', tz=tz)
assert result == expected
class TimestampOps(object):
goal_time = 0.2
def setup(self):
self.ts = Timestamp('2017-08-25 08:16:14')
self.ts_tz = Timestamp('2017-08-25 08:16:14', tz='US/Eastern')
dt = datetime.datetime(2016, 3, 27, 1)
self.tzinfo = pytz.timezone('CET').localize(dt, is_dst=False).tzinfo
self.ts2 = Timestamp(dt)
def time_replace_tz(self):
self.ts.replace(tzinfo=pytz.timezone('US/Eastern'))
def time_replace_across_dst(self):
self.ts2.replace(tzinfo=self.tzinfo)
def time_replace_None(self):
self.ts_tz.replace(tzinfo=None)
def time_to_pydatetime(self):
self.ts.to_pydatetime()
def time_to_pydatetime_tz(self):
self.ts_tz.to_pydatetime()
def test_replace_invalid_kwarg(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
ts = Timestamp("2016-01-01 09:00:00.000000123", tz=tz)
with pytest.raises(TypeError):
ts.replace(foo=5)
def evaluate_size_from_timestamp(start_timestamp: pd.Timestamp,
end_timestamp: pd.Timestamp,
level: IntervalLevel,
one_day_trading_minutes,
trade_day=None):
"""
given from timestamp,level,one_day_trading_minutes,this func evaluate size of kdata to current.
it maybe a little bigger than the real size for fetching all the kdata.
:param start_timestamp:
:type start_timestamp: pd.Timestamp
:param level:
:type level: IntervalLevel
:param one_day_trading_minutes:
:type one_day_trading_minutes: int
"""
# if not end_timestamp:
# end_timestamp = now_pd_timestamp()
# else:
# end_timestamp = to_pd_timestamp(end_timestamp)
time_delta = end_timestamp - to_pd_timestamp(start_timestamp)
one_day_trading_seconds = one_day_trading_minutes * 60
if level == IntervalLevel.LEVEL_1MON:
if trade_day is not None:
try:
size = int(math.ceil(trade_day.index(start_timestamp) / 22))
size = 0 if size == 0 else size + 1
return size
except ValueError as _:
if start_timestamp < trade_day[-1]:
return int(math.ceil(len(trade_day) / 22))
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days / 30))
if level == IntervalLevel.LEVEL_1WEEK:
if trade_day is not None:
try:
size = int(math.ceil(trade_day.index(start_timestamp) / 5))
size = 0 if size == 0 else size + 1
return size
except ValueError as _:
if start_timestamp < trade_day[-1]:
return int(math.ceil(len(trade_day) / 5))
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days / 7))
if level == IntervalLevel.LEVEL_1DAY:
if trade_day is not None and len(trade_day) > 0:
try:
return trade_day.index(start_timestamp)
except ValueError as _:
if start_timestamp < trade_day[-1]:
return len(trade_day)
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return time_delta.days
if level == IntervalLevel.LEVEL_1HOUR:
if trade_day is not None:
start_date = start_timestamp.replace(hour=0, minute=0, second=0)
try:
days = trade_day.index(start_date)
time = datetime.datetime.time(start_timestamp)
size = (days) * 4 + int(math.ceil(count_hours_from_day(time)))
return size
except ValueError as _:
if start_date < trade_day[-1]:
return len(trade_day) * 4
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days * 4 * 2))
if level == IntervalLevel.LEVEL_30MIN:
if trade_day is not None:
start_date = start_timestamp.replace(hour=0, minute=0, second=0)
try:
days = trade_day.index(start_date)
time = datetime.datetime.time(start_timestamp)
size = (days) * 4 * 2 + int(
math.ceil(count_mins_from_day(time) / 5))
return size
except ValueError as _:
if start_date < trade_day[-1]:
return len(trade_day) * 4 * 2
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days * 4 * 2))
if level == IntervalLevel.LEVEL_15MIN:
if trade_day is not None:
start_date = start_timestamp.replace(hour=0, minute=0, second=0)
try:
days = trade_day.index(start_date)
time = datetime.datetime.time(start_timestamp)
size = (days) * 4 * 4 + int(
math.ceil(count_mins_from_day(time) / 5))
return size
except ValueError as _:
if start_date < trade_day[-1]:
return len(trade_day) * 4 * 4
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days * 4 * 4))
if level == IntervalLevel.LEVEL_5MIN:
if trade_day is not None:
start_date = start_timestamp.replace(hour=0, minute=0, second=0)
try:
days = trade_day.index(start_date)
time = datetime.datetime.time(start_timestamp)
size = (days) * 4 * 12 + int(
math.ceil(count_mins_from_day(time) / 5))
return size
except ValueError as _:
if start_date < trade_day[-1]:
return len(trade_day) * 4 * 12
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days * 4 * 12))
if level == IntervalLevel.LEVEL_1MIN:
if trade_day is not None:
start_date = start_timestamp.replace(hour=0, minute=0, second=0)
try:
days = trade_day.index(start_date)
time = datetime.datetime.time(start_timestamp)
size = (days) * 4 * 60 + count_mins_from_day(time)
return size
except ValueError as _:
if start_date < trade_day[-1]:
return len(trade_day) * 4 * 60
# raise Exception("wrong start time:{}, error:{}".format(start_timestamp, e))
return int(math.ceil(time_delta.days * 4 * 60))
if time_delta.days > 0:
seconds = (time_delta.days + 1) * one_day_trading_seconds
return int(math.ceil(seconds / level.to_second()))
else:
seconds = time_delta.total_seconds()
return min(int(math.ceil(seconds / level.to_second())),
one_day_trading_seconds / level.to_second())
def test_replace_naive(self):
# GH#14621, GH#7825
ts = Timestamp("2016-01-01 09:00:00")
result = ts.replace(hour=0)
expected = Timestamp("2016-01-01 00:00:00")
assert result == expected
def test_replace_dst_border(self):
# Gh 7825
t = Timestamp('2013-11-3', tz='America/Chicago')
result = t.replace(hour=3)
expected = Timestamp('2013-11-3 03:00:00', tz='America/Chicago')
assert result == expected
def test_replace_tzinfo_equiv_tz_localize_none(self):
# GH#14621, GH#7825
# assert conversion to naive is the same as replacing tzinfo with None
ts = Timestamp('2013-11-03 01:59:59.999999-0400', tz='US/Eastern')
assert ts.tz_localize(None) == ts.replace(tzinfo=None)
def test_replace_preserves_nanos(self, tz):
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
result = ts.replace(hour=0)
expected = Timestamp('2016-01-01 00:00:00.000000123', tz=tz)
assert result == expected
def test_replace_dst_border(self):
# Gh 7825
t = Timestamp("2013-11-3", tz="America/Chicago")
result = t.replace(hour=3)
expected = Timestamp("2013-11-3 03:00:00", tz="America/Chicago")
assert result == expected
def test_replace_integer_args(self, tz_aware_fixture):
tz = tz_aware_fixture
# GH#14621, GH#7825
ts = Timestamp("2016-01-01 09:00:00.000000123", tz=tz)
with pytest.raises(ValueError):
ts.replace(hour=0.1)
def tsNextMonth(ts: pd.Timestamp) -> pd.Timestamp:
m = ts.month
if m == 12:
return ts.replace(year=ts.year + 1, month=1)
else:
return ts.replace(month=m + 1)
def __init__(self, initial_timestamp: pd.Timestamp):
self.start_time = initial_timestamp.replace(minute=0)
def _worker(self, exchange):
r = Rest()
storage = Storage(self.config)
for pair in self.config.backfill[exchange]:
try:
start = self.config.backfill[exchange][pair].start
while True:
end = storage.get_start_date(exchange, 'trades', pair)
if not all(e == end[0] for e in end):
raise InconsistentStorage(
"Stored data differs, cannot backfill")
end = end[0]
if end:
break
time.sleep(10)
end = Timestamp(end, unit='s')
end -= Timedelta(microseconds=1)
start = Timestamp(start)
if end <= Timestamp(start):
LOG.info(
"Data in storage is earlier than backfill start date for %s - %s",
exchange, pair)
continue
LOG.info("Backfill - Starting for %s - %s for range %s - %s",
exchange, pair, start, str(end))
# Backfill from end date to start date, 1 day at a time, in reverse order (from end -> start)
while start < end:
seg_start = end.replace(hour=0,
minute=0,
second=0,
microsecond=0,
nanosecond=0)
if start > seg_start:
seg_start = start
LOG.info("Backfill - Reading %s to %s for %s - %s",
seg_start, end, exchange, pair)
trades = []
try:
for t in r[exchange].trades(pair, str(seg_start),
str(end)):
trades.extend(t)
except Exception:
LOG.warning(
"Backfill - encountered error backfilling %s - %s, trying again...",
exchange,
pair,
exc_info=True)
time.sleep(300)
continue
if not trades:
end = seg_start - Timedelta(nanoseconds=1)
continue
storage.aggregate(trades)
storage.write(exchange, 'trades', pair, end.timestamp())
LOG.info("Backfill - Wrote %s to %s for %s - %s",
seg_start, end, exchange, pair)
end = seg_start - Timedelta(nanoseconds=1)
LOG.info("Backfill for %s - %s completed", exchange, pair)
except Exception:
LOG.error("Backfill failed for %s - %s",
exchange,
pair,
exc_info=True)
def make_end_of_day(timestamp: pd.Timestamp) -> pd.Timestamp:
return timestamp.replace(hour=23, minute=59, second=59)
def test_replace_invalid_kwarg(self, tz):
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
with pytest.raises(TypeError):
ts.replace(foo=5)
def test_replace_tzinfo_equiv_tz_localize_none(self):
# GH#14621, GH#7825
# assert conversion to naive is the same as replacing tzinfo with None
ts = Timestamp("2013-11-03 01:59:59.999999-0400", tz="US/Eastern")
assert ts.tz_localize(None) == ts.replace(tzinfo=None)
def test_replace_invalid_kwarg(self, tz):
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
with pytest.raises(TypeError):
ts.replace(foo=5)
def test_replace_integer_args(self, tz):
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
with pytest.raises(ValueError):
ts.replace(hour=0.1)
def test_replace_dst_border(self):
# Gh 7825
t = Timestamp('2013-11-3', tz='America/Chicago')
result = t.replace(hour=3)
expected = Timestamp('2013-11-3 03:00:00', tz='America/Chicago')
assert result == expected
def test_replace_integer_args(self, tz):
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00.000000123', tz=tz)
with pytest.raises(ValueError):
ts.replace(hour=0.1)
def test_replace_naive(self):
# GH#14621, GH#7825
ts = Timestamp('2016-01-01 09:00:00')
result = ts.replace(hour=0)
expected = Timestamp('2016-01-01 00:00:00')
assert result == expected