def test_timezone_conversion(self):
est: time.Timezone = time.timezone('US/Eastern')
fmt = '%a, %e %b %Y %H:%M:%S %z (%Z)'
# A non-transition where the civil time is unique.
nov01 = time.CivilTime(2013, 11, 1, 8, 30, 0)
nov01_t = time.FromCivil(nov01, est)
self.assertEqual("Fri, 1 Nov 2013 08:30:00 -0400 (EDT)",
time.FormatTime(nov01_t, est, fmt))
# A Spring DST transition, when there is a gap in civil time
# and we prefer the later of the possible interpretations of a
# non-existent time.
mar13 = time.CivilTime(2011, 3, 13, 2, 15, 0)
mar_t = time.FromCivil(mar13, est)
self.assertEqual("Sun, 13 Mar 2011 03:15:00 -0400 (EDT)",
time.FormatTime(mar_t, est, fmt))
# A Fall DST transition, when civil times are repeated and
# we prefer the earlier of the possible interpretations of an
# ambiguous time.
nov06 = time.CivilTime(2011, 11, 6, 1, 15, 0)
nov_t = time.FromCivil(nov06, est)
self.assertEqual("Sun, 6 Nov 2011 01:15:00 -0400 (EDT)",
time.FormatTime(nov_t, est, fmt))
# Check that (time_t) -1 is handled correctly.
minus1 = time.CivilTime(1969, 12, 31, 18, 59, 59)
minus1_t = time.FromCivil(minus1, est)
self.assertEqual("Wed, 31 Dec 1969 18:59:59 -0500 (EST)",
time.FormatTime(minus1_t, est, fmt))
self.assertEqual("Wed, 31 Dec 1969 23:59:59 +0000 (UTC)",
time.FormatTime(minus1_t, time.UTC, fmt))
def test_get_quotes_success(self, mock_get, mock_now):
mock_get.return_value = _make_response(
200, """{
"SPY": {
"chart": [
{"date":"2020-03-05","close":319.69,"volume":242964067,"change":0,"changePercent":0,"changeOverTime":0},
{"date":"2020-03-06","close":308.48,"volume":300862938,"change":-9.12,"changePercent":-2.9921,"changeOverTime":-0.029457},
{"date":"2020-03-09","close":301.01,"volume":123123131,"change":-7.47,"changePercent":-2.4216,"changeOverTime":-0.01311}
]
}
}""")
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 3, 11))
results = self.importer.get_data('SPY', time.CivilTime(2020, 3, 5),
time.CivilTime(2020, 3, 7))
_, kwargs = mock_get.call_args
self.assertEqual(kwargs['params']['symbols'], 'SPY')
self.assertEqual(kwargs['params']['range'], '1m')
assert_that(
results[data_type_pb2.DataType.CLOSE_PRICE],
contains(
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: CLOSE_PRICE
value: 319.69
timestamp {{ seconds: {time.as_seconds(2020, 3, 5)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
"""),
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: CLOSE_PRICE
value: 308.48
timestamp {{ seconds: {time.as_seconds(2020, 3, 6)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
assert_that(
results[data_type_pb2.DataType.VOLUME],
contains(
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: VOLUME
value: 242964067.0
timestamp {{ seconds: {time.as_seconds(2020, 3, 5)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
"""),
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: VOLUME
value: 300862938.0
timestamp {{ seconds: {time.as_seconds(2020, 3, 6)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
def test_default_time_format(self):
t = time.FromCivil(time.CivilTime(2015, 2, 3, 4, 5, 6))
self.assertEqual("2015-02-03T04:05:06+00:00", time.FormatTime(t))
t = time.FromCivil(time.CivilTime(2015, 2, 3, 4, 5))
self.assertEqual("2015-02-03T04:05:00+00:00", time.FormatTime(t))
t = time.FromCivil(time.CivilTime(2015, 2, 3, 4))
self.assertEqual("2015-02-03T04:00:00+00:00", time.FormatTime(t))
t = time.FromCivil(time.CivilTime(2015, 2, 3))
self.assertEqual("2015-02-03T00:00:00+00:00", time.FormatTime(t))
def test_format_civil_time(self):
ct = time.CivilTime(1970, 1, 1)
self.assertEqual("1970-01-01T00:00:00", time.FormatCivilTime(ct))
ct = time.CivilTime(2015, 1, 2, 3, 4, 5)
self.assertEqual("2015-01-02T03:04:05", time.FormatCivilTime(ct))
ct = time.ToCivil(time.Time(1234567890 * 1e9))
self.assertEqual("2009-02-13T23:31:30", time.FormatCivilTime(ct))
est: time.Timezone = time.timezone('US/Eastern')
ct = time.ToCivil(time.Time(1234567890 * 1e9), est)
self.assertEqual("2009-02-13T18:31:30", time.FormatCivilTime(ct))
def test_from_civil(self):
fmt = "%a, %e %b %Y %H:%M:%S %z (%Z)"
# Check that we're counting leap years correctly.
t = time.FromCivil(time.CivilTime(1900, 2, 28, 23, 59, 59), time.UTC)
self.assertEqual("Wed, 28 Feb 1900 23:59:59 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
t = time.FromCivil(time.CivilTime(1900, 3, 1, 0, 0, 0), time.UTC)
self.assertEqual("Thu, 1 Mar 1900 00:00:00 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
t = time.FromCivil(time.CivilTime(2000, 2, 29, 23, 59, 59), time.UTC)
self.assertEqual("Tue, 29 Feb 2000 23:59:59 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
t = time.FromCivil(time.CivilTime(2000, 3, 1, 0, 0, 0), time.UTC)
self.assertEqual("Wed, 1 Mar 2000 00:00:00 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
def test_ema_20_initial_calculation(self):
# INTC 2019-12-03 to 2019-12-31
price_data = [
56.07, 56.02, 56.08, 56.81, 56.53, 56.59, 57.07, 57.55, 57.79, 57.70,
57.23, 57.17, 57.96, 58.95, 59.23, 59.41, 59.82, 60.08, 59.62, 59.85,
] # yapf: disable
date = time.CivilTime(2019, 12, 31)
close_prices = [
_make_close_price(value, time.FromCivil(d))
for value, d in zip(price_data, trading_days.get_last_n(date, 20))
]
# Note: this is different than the actual EMA20 for Intel on 2019/12/31,
# which is 58.46, because EMA accounts for *all* past data
assert_that(
ema.make_ema_20d_producer().calculate(close_prices),
equals_proto(f"""
symbol: "TEST"
data_space: STOCK_DATA
data_type: EMA_20D
value: 58.255796513052346
timestamp {{
seconds: {time.as_seconds(2019, 12, 31)}
}}"""))
def test_produce_source_calc_fn_multiple_symbols(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = [
*get_close_prices_for_ema20(t, 'TEST'),
*get_close_prices_for_ema20(t, 'IBM')
]
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_source_calc_fn(DataType.EMA_20D, t))
assert_that(
out,
equal_to([
expected_ema20d_test(),
text_format.Parse(
f"""
symbol: "IBM"
data_space: STOCK_DATA
data_type: EMA_20D
value: 58.255796513052346
timestamp {{
seconds: {time.as_seconds(2019, 12, 31)}
}}""", DataEntry())
]))
def test_get_quotes_backend_failure(self, mock_get, mock_now):
mock_get.return_value = _make_response(501, '', 'Server Unavailable')
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 12, 31))
with self.assertRaisesRegex(errors.InternalError,
'501 Server Error: Server Unavailable'):
self.importer.get_data('SPY')
def test_produce_source_calc_fn(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = get_close_prices_for_ema20(t, 'TEST')
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_source_calc_fn(DataType.EMA_20D, t))
assert_that(out, equal_to([expected_ema20d_test()]))
def test_produce_recursive_calc_fn_with_swapped_inputs(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = reversed(get_recursive_inputs_for_ema20(t, 'TEST'))
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_recursive_calc_fn(DataType.EMA_20D, t))
assert_that(out, equal_to([expected_recursive_ema20d_test()]))
def test_raise_when_inputs_do_not_meet_accpeted_input_shapes(self):
date = time.CivilTime(2017, 10, 10)
close_prices = [
_make_close_price(200.0, time.FromCivil(d))
for d in trading_days.get_last_n(date, 19)
]
with self.assertRaisesRegex(
errors.InvalidArgumentError,
'Expecting data with input shape: data_type: CLOSE_PRICE'):
ema.make_ema_20d_producer().calculate(close_prices)
def test_produce_recursive_calc_fn_with_extra_data(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = [
*get_recursive_inputs_for_ema20(t, 'TEST'),
*get_close_prices_for_ema20(t, 'IBM')
]
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_recursive_calc_fn(DataType.EMA_20D, t))
assert_that(out, equal_to([expected_recursive_ema20d_test()]))
def test_is_trading_day(self):
finder = trading_days.get_trading_day_finder()
self.assertFalse(finder.is_trading_day(time.CivilTime(2017, 1, 2)))
self.assertTrue(finder.is_trading_day(time.CivilTime(2017, 1, 3)))
self.assertTrue(finder.is_trading_day(time.CivilTime(2018, 7, 3)))
self.assertFalse(finder.is_trading_day(time.CivilTime(2018, 7, 4)))
self.assertTrue(finder.is_trading_day(time.CivilTime(2020, 5, 22)))
self.assertFalse(finder.is_trading_day(time.CivilTime(2020, 5, 25)))
def test_parse_civil_time(self):
self.assertEqual(time.CivilTime(2015, 1, 2, 3, 4, 5),
time.ParseCivilTime("2015-01-02T03:04:05"))
self.assertEqual(time.CivilTime(2015, 1, 2, 3, 4),
time.ParseCivilTime("2015-01-02T03:04:00"))
self.assertEqual(time.CivilTime(2015, 1, 2, 3),
time.ParseCivilTime("2015-01-02T03:00:00"))
self.assertEqual(time.CivilTime(2015, 1, 2, 3),
time.ParseCivilTime("2015-01-02T03:00"))
self.assertEqual(time.CivilTime(2015, 1, 2),
time.ParseCivilTime("2015-01-02T00:00:00"))
self.assertEqual(time.CivilTime(2015, 1, 2),
time.ParseCivilTime("2015-01-02T00:00"))
self.assertEqual(time.CivilTime(2015, 1, 2),
time.ParseCivilTime("2015-01-02"))
def test_ema_20_for_constant_list(self):
date = time.CivilTime(2017, 10, 10)
close_prices = [
_make_close_price(200.0, time.FromCivil(d))
for d in trading_days.get_last_n(date, 20)
]
assert_that(
ema.make_ema_20d_producer().calculate(close_prices),
equals_proto(f"""
symbol: "TEST"
data_space: STOCK_DATA
data_type: EMA_20D
value: 200.0
timestamp {{
seconds: {time.as_seconds(2017, 10, 10)}
}}"""))
def test_making_series_input_shape_account_for_trading_days(self):
got = input_util.series_source_inputs_shape(
source_calc_type=_DataType.Enum.CLOSE_PRICE,
t=time.FromCivil(time.CivilTime(2017, 12, 26)),
time_spec=calc.CalcTimeSpecs(num_periods=2,
period_length=time.Hours(24)))
assert_that(
got,
contains(
equals_proto(f"""
data_type: CLOSE_PRICE
timestamp {{
seconds: {time.as_seconds(2017, 12, 22)}
}}"""),
equals_proto(f"""
data_type: CLOSE_PRICE
timestamp {{
seconds: {time.as_seconds(2017, 12, 26)}
}}""")))
def test_get_quotes_default_single_day(self, mock_get, mock_now):
mock_get.return_value = _make_response(
200, """{
"ABC": {
"chart": [
{"date":"2020-03-11","close":300.00,"volume":231231312,"change":-0,"changePercent": 0.0,"changeOverTime":-0.04321}
]
}
}""")
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 3, 11))
results = self.importer.get_data('ABC')
_, kwargs = mock_get.call_args
self.assertEqual(kwargs['params']['symbols'], 'ABC')
self.assertEqual(kwargs['params']['range'], '5d')
assert_that(
results[data_type_pb2.DataType.CLOSE_PRICE],
contains(
equals_proto(f"""
symbol: "ABC"
data_space: STOCK_DATA
data_type: CLOSE_PRICE
value: 300.00
timestamp {{ seconds: {time.as_seconds(2020, 3, 11)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
assert_that(
results[data_type_pb2.DataType.VOLUME],
contains(
equals_proto(f"""
symbol: "ABC"
data_space: STOCK_DATA
data_type: VOLUME
value: 231231312.0
timestamp {{ seconds: {time.as_seconds(2020, 3, 11)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
def test_ema20_recursive_calculation(self):
date = time.FromCivil(time.CivilTime(2020, 1, 1))
close_now = _make_close_price(60.84, date)
ema_last = _DataEntry(
symbol='TEST',
data_space=_DataEntry.STOCK_DATA,
data_type=data_type_pb2.DataType.EMA_20D,
value=58.46,
timestamp=time_util.from_time(date - time.Hours(24)),
)
assert_that(
ema.make_ema_20d_producer().calculate((ema_last, close_now)),
equals_proto(f"""
symbol: "TEST"
data_space: STOCK_DATA
data_type: EMA_20D
value: 58.68666666666667
timestamp {{
seconds: {time.as_seconds(2020, 1, 1)}
}}"""))
def test_get_next_n_with_input_date(self):
finder = trading_days.get_trading_day_finder()
self.assertSequenceEqual(
finder.get_next_n(time.CivilTime(2020, 5, 25),
1,
include_input_date=True),
[time.CivilTime(2020, 5, 26)])
self.assertSequenceEqual(
finder.get_next_n(time.CivilTime(2020, 5, 26),
1,
include_input_date=True),
[time.CivilTime(2020, 5, 26)])
self.assertSequenceEqual(
finder.get_next_n(time.CivilTime(2020, 5, 26),
6,
include_input_date=True), [
time.CivilTime(2020, 5, 26),
time.CivilTime(2020, 5, 27),
time.CivilTime(2020, 5, 28),
time.CivilTime(2020, 5, 29),
time.CivilTime(2020, 6, 1),
time.CivilTime(2020, 6, 2),
])
def test_unix_epoch(self):
epoch = time.FromCivil(time.CivilTime(1970, 1, 1, 0, 0, 0))
self.assertEqual(epoch, time.UnixEpoch())
self.assertEqual(epoch - time.UnixEpoch(), time.ZeroDuration())
def test_get_quotes_invalid_dates(self, mock_get, mock_now):
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 12, 31))
with self.assertRaisesRegex(
errors.InvalidArgumentError,
'end_date 2021-01-01T00:00:00 must be in the past'):
self.importer.get_data('SPY', end_date=time.CivilTime(2021, 1, 1))
def test_civil_time_relations(self):
self._test_relational(time.CivilTime(2014, 1, 1, 0, 0, 0),
time.CivilTime(2015, 1, 1, 0, 0, 0))
self._test_relational(time.CivilTime(2014, 1, 1, 0, 0, 0),
time.CivilTime(2014, 2, 1, 0, 0, 0))
self._test_relational(time.CivilTime(2014, 1, 1, 0, 0, 0),
time.CivilTime(2014, 1, 2, 0, 0, 0))
self._test_relational(time.CivilTime(2014, 1, 1, 0, 0, 0),
time.CivilTime(2014, 1, 1, 1, 0, 0))
self._test_relational(time.CivilTime(2014, 1, 1, 1, 0, 0),
time.CivilTime(2014, 1, 1, 1, 1, 0))
self._test_relational(time.CivilTime(2014, 1, 1, 1, 1, 0),
time.CivilTime(2014, 1, 1, 1, 1, 1))
self._test_relational(time.CivilTime(2014, 1, 1),
time.CivilTime(2014, 1, 1, 1, 1))
self._test_relational(time.CivilTime(2014, 1, 1),
time.CivilTime(2014, 2, 1))
def test_get_last_n(self):
finder = trading_days.get_trading_day_finder()
# 2020-05-25 is not a traiding date.
self.assertSequenceEqual(
finder.get_last_n(time.CivilTime(2020, 5, 25), 1),
[time.CivilTime(2020, 5, 22)])
self.assertSequenceEqual(
finder.get_last_n(time.CivilTime(2020, 5, 22), 1),
[time.CivilTime(2020, 5, 22)])
self.assertSequenceEqual(
finder.get_last_n(time.CivilTime(2020, 5, 25), 6), [
time.CivilTime(2020, 5, 15),
time.CivilTime(2020, 5, 18),
time.CivilTime(2020, 5, 19),
time.CivilTime(2020, 5, 20),
time.CivilTime(2020, 5, 21),
time.CivilTime(2020, 5, 22),
])
def test_get_last_n_without_input_date(self):
finder = trading_days.get_trading_day_finder()
self.assertSequenceEqual(
finder.get_last_n(time.CivilTime(2020, 5, 25),
1,
include_input_date=False),
[time.CivilTime(2020, 5, 22)])
self.assertSequenceEqual(
finder.get_last_n(time.CivilTime(2020, 5, 22),
1,
include_input_date=False),
[time.CivilTime(2020, 5, 21)])
self.assertSequenceEqual(
finder.get_last_n(time.CivilTime(2020, 5, 22),
6,
include_input_date=False), [
time.CivilTime(2020, 5, 14),
time.CivilTime(2020, 5, 15),
time.CivilTime(2020, 5, 18),
time.CivilTime(2020, 5, 19),
time.CivilTime(2020, 5, 20),
time.CivilTime(2020, 5, 21),
])
def test_get_next_n(self):
finder = trading_days.get_trading_day_finder()
self.assertSequenceEqual(
finder.get_next_n(time.CivilTime(2020, 5, 25), 1),
[time.CivilTime(2020, 5, 26)])
self.assertSequenceEqual(
finder.get_next_n(time.CivilTime(2020, 5, 26), 1),
[time.CivilTime(2020, 5, 27)])
self.assertSequenceEqual(
finder.get_next_n(time.CivilTime(2020, 5, 23), 6), [
time.CivilTime(2020, 5, 26),
time.CivilTime(2020, 5, 27),
time.CivilTime(2020, 5, 28),
time.CivilTime(2020, 5, 29),
time.CivilTime(2020, 6, 1),
time.CivilTime(2020, 6, 2),
])
