def test_timezone_conversion(self):
est: time.Timezone = time.timezone('US/Eastern')
fmt = '%a, %e %b %Y %H:%M:%S %z (%Z)'
# A non-transition where the civil time is unique.
nov01 = time.CivilTime(2013, 11, 1, 8, 30, 0)
nov01_t = time.FromCivil(nov01, est)
self.assertEqual("Fri, 1 Nov 2013 08:30:00 -0400 (EDT)",
time.FormatTime(nov01_t, est, fmt))
# A Spring DST transition, when there is a gap in civil time
# and we prefer the later of the possible interpretations of a
# non-existent time.
mar13 = time.CivilTime(2011, 3, 13, 2, 15, 0)
mar_t = time.FromCivil(mar13, est)
self.assertEqual("Sun, 13 Mar 2011 03:15:00 -0400 (EDT)",
time.FormatTime(mar_t, est, fmt))
# A Fall DST transition, when civil times are repeated and
# we prefer the earlier of the possible interpretations of an
# ambiguous time.
nov06 = time.CivilTime(2011, 11, 6, 1, 15, 0)
nov_t = time.FromCivil(nov06, est)
self.assertEqual("Sun, 6 Nov 2011 01:15:00 -0400 (EDT)",
time.FormatTime(nov_t, est, fmt))
# Check that (time_t) -1 is handled correctly.
minus1 = time.CivilTime(1969, 12, 31, 18, 59, 59)
minus1_t = time.FromCivil(minus1, est)
self.assertEqual("Wed, 31 Dec 1969 18:59:59 -0500 (EST)",
time.FormatTime(minus1_t, est, fmt))
self.assertEqual("Wed, 31 Dec 1969 23:59:59 +0000 (UTC)",
time.FormatTime(minus1_t, time.UTC, fmt))
def test_default_time_format(self):
t = time.FromCivil(time.CivilTime(2015, 2, 3, 4, 5, 6))
self.assertEqual("2015-02-03T04:05:06+00:00", time.FormatTime(t))
t = time.FromCivil(time.CivilTime(2015, 2, 3, 4, 5))
self.assertEqual("2015-02-03T04:05:00+00:00", time.FormatTime(t))
t = time.FromCivil(time.CivilTime(2015, 2, 3, 4))
self.assertEqual("2015-02-03T04:00:00+00:00", time.FormatTime(t))
t = time.FromCivil(time.CivilTime(2015, 2, 3))
self.assertEqual("2015-02-03T00:00:00+00:00", time.FormatTime(t))
def test_from_civil(self):
fmt = "%a, %e %b %Y %H:%M:%S %z (%Z)"
# Check that we're counting leap years correctly.
t = time.FromCivil(time.CivilTime(1900, 2, 28, 23, 59, 59), time.UTC)
self.assertEqual("Wed, 28 Feb 1900 23:59:59 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
t = time.FromCivil(time.CivilTime(1900, 3, 1, 0, 0, 0), time.UTC)
self.assertEqual("Thu, 1 Mar 1900 00:00:00 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
t = time.FromCivil(time.CivilTime(2000, 2, 29, 23, 59, 59), time.UTC)
self.assertEqual("Tue, 29 Feb 2000 23:59:59 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
t = time.FromCivil(time.CivilTime(2000, 3, 1, 0, 0, 0), time.UTC)
self.assertEqual("Wed, 1 Mar 2000 00:00:00 +0000 (UTC)",
time.FormatTime(t, time.UTC, fmt))
def test_get_quotes_backend_failure(self, mock_get, mock_now):
mock_get.return_value = _make_response(501, '', 'Server Unavailable')
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 12, 31))
with self.assertRaisesRegex(errors.InternalError,
'501 Server Error: Server Unavailable'):
self.importer.get_data('SPY')
def test_produce_source_calc_fn_multiple_symbols(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = [
*get_close_prices_for_ema20(t, 'TEST'),
*get_close_prices_for_ema20(t, 'IBM')
]
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_source_calc_fn(DataType.EMA_20D, t))
assert_that(
out,
equal_to([
expected_ema20d_test(),
text_format.Parse(
f"""
symbol: "IBM"
data_space: STOCK_DATA
data_type: EMA_20D
value: 58.255796513052346
timestamp {{
seconds: {time.as_seconds(2019, 12, 31)}
}}""", DataEntry())
]))
def test_ema_20_initial_calculation(self):
# INTC 2019-12-03 to 2019-12-31
price_data = [
56.07, 56.02, 56.08, 56.81, 56.53, 56.59, 57.07, 57.55, 57.79, 57.70,
57.23, 57.17, 57.96, 58.95, 59.23, 59.41, 59.82, 60.08, 59.62, 59.85,
] # yapf: disable
date = time.CivilTime(2019, 12, 31)
close_prices = [
_make_close_price(value, time.FromCivil(d))
for value, d in zip(price_data, trading_days.get_last_n(date, 20))
]
# Note: this is different than the actual EMA20 for Intel on 2019/12/31,
# which is 58.46, because EMA accounts for *all* past data
assert_that(
ema.make_ema_20d_producer().calculate(close_prices),
equals_proto(f"""
symbol: "TEST"
data_space: STOCK_DATA
data_type: EMA_20D
value: 58.255796513052346
timestamp {{
seconds: {time.as_seconds(2019, 12, 31)}
}}"""))
def test_get_quotes_success(self, mock_get, mock_now):
mock_get.return_value = _make_response(
200, """{
"SPY": {
"chart": [
{"date":"2020-03-05","close":319.69,"volume":242964067,"change":0,"changePercent":0,"changeOverTime":0},
{"date":"2020-03-06","close":308.48,"volume":300862938,"change":-9.12,"changePercent":-2.9921,"changeOverTime":-0.029457},
{"date":"2020-03-09","close":301.01,"volume":123123131,"change":-7.47,"changePercent":-2.4216,"changeOverTime":-0.01311}
]
}
}""")
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 3, 11))
results = self.importer.get_data('SPY', time.CivilTime(2020, 3, 5),
time.CivilTime(2020, 3, 7))
_, kwargs = mock_get.call_args
self.assertEqual(kwargs['params']['symbols'], 'SPY')
self.assertEqual(kwargs['params']['range'], '1m')
assert_that(
results[data_type_pb2.DataType.CLOSE_PRICE],
contains(
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: CLOSE_PRICE
value: 319.69
timestamp {{ seconds: {time.as_seconds(2020, 3, 5)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
"""),
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: CLOSE_PRICE
value: 308.48
timestamp {{ seconds: {time.as_seconds(2020, 3, 6)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
assert_that(
results[data_type_pb2.DataType.VOLUME],
contains(
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: VOLUME
value: 242964067.0
timestamp {{ seconds: {time.as_seconds(2020, 3, 5)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
"""),
equals_proto(f"""
symbol: "SPY"
data_space: STOCK_DATA
data_type: VOLUME
value: 300862938.0
timestamp {{ seconds: {time.as_seconds(2020, 3, 6)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
def test_produce_source_calc_fn(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = get_close_prices_for_ema20(t, 'TEST')
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_source_calc_fn(DataType.EMA_20D, t))
assert_that(out, equal_to([expected_ema20d_test()]))
def test_produce_recursive_calc_fn_with_swapped_inputs(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = reversed(get_recursive_inputs_for_ema20(t, 'TEST'))
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_recursive_calc_fn(DataType.EMA_20D, t))
assert_that(out, equal_to([expected_recursive_ema20d_test()]))
def test_raise_when_inputs_do_not_meet_accpeted_input_shapes(self):
date = time.CivilTime(2017, 10, 10)
close_prices = [
_make_close_price(200.0, time.FromCivil(d))
for d in trading_days.get_last_n(date, 19)
]
with self.assertRaisesRegex(
errors.InvalidArgumentError,
'Expecting data with input shape: data_type: CLOSE_PRICE'):
ema.make_ema_20d_producer().calculate(close_prices)
def test_produce_recursive_calc_fn_with_extra_data(self):
t = time.FromCivil(time.CivilTime(2019, 12, 31))
inputs = [
*get_recursive_inputs_for_ema20(t, 'TEST'),
*get_close_prices_for_ema20(t, 'IBM')
]
with TestPipeline() as p:
out = (p \
| beam.Create(inputs) \
| produce_recursive_calc_fn(DataType.EMA_20D, t))
assert_that(out, equal_to([expected_recursive_ema20d_test()]))
def get_close_prices_for_ema20(t: time.Time, symbol: str) -> List[DataEntry]:
# INTC 2019-12-03 to 2019-12-31
price_data = [
56.07, 56.02, 56.08, 56.81, 56.53, 56.59, 57.07, 57.55, 57.79, 57.70,
57.23, 57.17, 57.96, 58.95, 59.23, 59.41, 59.82, 60.08, 59.62, 59.85,
] # yapf: disable
close_prices = [
_make_close_price(symbol, value, time.FromCivil(d))
for value, d in zip(price_data,
trading_days.get_last_n(time.ToCivil(t), 20))
]
random.shuffle(close_prices)
return close_prices
def recursive_inputs_shape(
base_calc_type: DataType.Enum, incr_calc_type: DataType.Enum,
t: time.Time, time_spec: calc.CalcTimeSpecs) -> calc.RecursiveInputs:
# TODO: same here, think about how to get it work with period other than
# days. e.g week-based moving averages.
the_day_before = trading_days.get_last_n(time.ToCivil(t),
1,
include_input_date=False)[0]
return (DataEntry(
data_type=base_calc_type,
timestamp=time_util.from_time(time.FromCivil(the_day_before)),
), DataEntry(
data_type=incr_calc_type,
timestamp=time_util.from_time(t),
))
def test_ema_20_for_constant_list(self):
date = time.CivilTime(2017, 10, 10)
close_prices = [
_make_close_price(200.0, time.FromCivil(d))
for d in trading_days.get_last_n(date, 20)
]
assert_that(
ema.make_ema_20d_producer().calculate(close_prices),
equals_proto(f"""
symbol: "TEST"
data_space: STOCK_DATA
data_type: EMA_20D
value: 200.0
timestamp {{
seconds: {time.as_seconds(2017, 10, 10)}
}}"""))
def series_source_inputs_shape(
source_calc_type: DataType.Enum, t: time.Time,
time_spec: calc.CalcTimeSpecs) -> calc.SourceInputs:
# TODO: right now this is still no properly generalized. For time span
# longer than a day, they should all work. However for day and shorter span,
# they need to be checked against trading days and trading hours.
# NOTE: both FromCivil and ToCivil assumes UTC, which later might
# be updated to take into account the exchange timezone of a
# security.
timestamps = [
time.FromCivil(day) for day in trading_days.get_last_n(
time.ToCivil(t), time_spec.num_periods)
]
return [
DataEntry(
data_type=source_calc_type,
timestamp=time_util.from_time(timestmap),
) for timestmap in timestamps
]
def test_making_series_input_shape_account_for_trading_days(self):
got = input_util.series_source_inputs_shape(
source_calc_type=_DataType.Enum.CLOSE_PRICE,
t=time.FromCivil(time.CivilTime(2017, 12, 26)),
time_spec=calc.CalcTimeSpecs(num_periods=2,
period_length=time.Hours(24)))
assert_that(
got,
contains(
equals_proto(f"""
data_type: CLOSE_PRICE
timestamp {{
seconds: {time.as_seconds(2017, 12, 22)}
}}"""),
equals_proto(f"""
data_type: CLOSE_PRICE
timestamp {{
seconds: {time.as_seconds(2017, 12, 26)}
}}""")))
def test_get_quotes_default_single_day(self, mock_get, mock_now):
mock_get.return_value = _make_response(
200, """{
"ABC": {
"chart": [
{"date":"2020-03-11","close":300.00,"volume":231231312,"change":-0,"changePercent": 0.0,"changeOverTime":-0.04321}
]
}
}""")
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 3, 11))
results = self.importer.get_data('ABC')
_, kwargs = mock_get.call_args
self.assertEqual(kwargs['params']['symbols'], 'ABC')
self.assertEqual(kwargs['params']['range'], '5d')
assert_that(
results[data_type_pb2.DataType.CLOSE_PRICE],
contains(
equals_proto(f"""
symbol: "ABC"
data_space: STOCK_DATA
data_type: CLOSE_PRICE
value: 300.00
timestamp {{ seconds: {time.as_seconds(2020, 3, 11)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
assert_that(
results[data_type_pb2.DataType.VOLUME],
contains(
equals_proto(f"""
symbol: "ABC"
data_space: STOCK_DATA
data_type: VOLUME
value: 231231312.0
timestamp {{ seconds: {time.as_seconds(2020, 3, 11)} }}
updated_at {{ seconds: {time.as_seconds(2020, 3, 11)} }}
""")))
def test_ema20_recursive_calculation(self):
date = time.FromCivil(time.CivilTime(2020, 1, 1))
close_now = _make_close_price(60.84, date)
ema_last = _DataEntry(
symbol='TEST',
data_space=_DataEntry.STOCK_DATA,
data_type=data_type_pb2.DataType.EMA_20D,
value=58.46,
timestamp=time_util.from_time(date - time.Hours(24)),
)
assert_that(
ema.make_ema_20d_producer().calculate((ema_last, close_now)),
equals_proto(f"""
symbol: "TEST"
data_space: STOCK_DATA
data_type: EMA_20D
value: 58.68666666666667
timestamp {{
seconds: {time.as_seconds(2020, 1, 1)}
}}"""))
def run(date: time.CivilTime, calc_type: DataType.Enum,
is_recursive_calc: bool):
t = time.FromCivil(date)
calc_producer = calc_registry.get_calc_producer(calc_type)
inputs_shape = calc_producer.recursive_inputs_shape(
t) if is_recursive_calc else calc_producer.source_inputs_shape(t)
mongo_read_filter = {
'timestamp': {
'$gte': time_util.to_civil(inputs_shape[0].timestamp),
'$lte': time_util.to_civil(inputs_shape[-1].timestamp)
}
}
with beam.Pipeline() as p:
(
p \
| mongo_util.ReadDataFromMongoDB(
mongo_util.default_read_option(mongo_read_filter))
| "ProduceCalcs" >> produce_calc_fn.produce_calc_fn(
calc_producer, inputs_shape) \
| mongo_util.WriteDataToMongoDB(mongo_util.default_write_option())
)
def test_unix_epoch(self):
epoch = time.FromCivil(time.CivilTime(1970, 1, 1, 0, 0, 0))
self.assertEqual(epoch, time.UnixEpoch())
self.assertEqual(epoch - time.UnixEpoch(), time.ZeroDuration())
def from_civil(ct: time.CivilTime) -> timestamp_pb2.Timestamp:
return from_time(time.FromCivil(ct))
def test_get_quotes_invalid_dates(self, mock_get, mock_now):
mock_now.return_value = time.FromCivil(time.CivilTime(2020, 12, 31))
with self.assertRaisesRegex(
errors.InvalidArgumentError,
'end_date 2021-01-01T00:00:00 must be in the past'):
self.importer.get_data('SPY', end_date=time.CivilTime(2021, 1, 1))
