def get_beginning_of_next_trading_day(ref_time):
ref_day = posix_to_datetime(ref_time).date()
schedule = get_trading_calendar(ref_day, ref_day)
while schedule.empty:
ref_day += timedelta(days=1)
schedule = get_trading_calendar(ref_day, ref_day)
start_day, _ = get_schedule_start_and_end(schedule)
return start_day
def test_time_handlers(self):
posix_time = 1590165714.1528566
actual_date = dt(2020, 5, 22, 12, 41, 54, 152857)
localizer = pytz.timezone(TIMEZONE)
localized_date = localizer.localize(actual_date)
self.assertEqual(posix_to_datetime(posix_time), localized_date)
self.assertAlmostEqual(posix_time, datetime_to_posix(localized_date),
0)
mexico_date = actual_date.replace(hour=11)
localizer = pytz.timezone("America/Mexico_City")
localized_date = localizer.localize(mexico_date)
self.assertAlmostEqual(posix_time, datetime_to_posix(localized_date),
0)
# Pre-stage all of the mocked current time values that will be called sequentially in the tests below.
# ----------------------------------------------------------------------------------------------------
with patch('backend.logic.base.time') as current_time_mock:
# Check during trading day just one second before and after open/close
schedule = get_trading_calendar(actual_date, actual_date)
start_day, end_day = [
datetime_to_posix(x)
for x in schedule.iloc[0][["market_open", "market_close"]]
]
current_time_mock.time.side_effect = [
posix_time, # during trading day
posix_time + 8 * 60 * 60, # 8 hours later--after trading day
1608908400, # Christmas 2020, Friday, 11am in NYC. We want to verify that we're accounting for holidays
start_day - 1, # one second before trading day
(start_day + end_day) /
2, # right in the middle of trading day
end_day + 1 # one second after trading day
]
self.assertTrue(during_trading_day())
self.assertFalse(during_trading_day())
self.assertFalse(during_trading_day())
self.assertFalse(during_trading_day())
self.assertTrue(during_trading_day())
self.assertFalse(during_trading_day())
# Finally, just double-check that the real-time, default invocation works as expected
posix_now = time.time()
nyc_now = posix_to_datetime(posix_now)
schedule = get_trading_calendar(nyc_now, nyc_now)
during_trading = False
if not schedule.empty:
start_day, end_day = [
datetime_to_posix(x)
for x in schedule.iloc[0][["market_open", "market_close"]]
]
during_trading = start_day <= posix_now <= end_day
# FYI: there is a non-zero chance that this test will fail at exactly the beginning or end of a trading day
self.assertEqual(during_trading, during_trading_day())
def test_task_caching(self):
rds.flushall()
test_time = posix_to_datetime(time.time()).date()
start = time.time()
_ = get_trading_calendar(test_time, test_time)
time1 = time.time() - start
start = time.time()
_ = get_trading_calendar(test_time, test_time)
time2 = time.time() - start
self.assertLess(time2, time1 / 2) # "2" is a hueristic for 'substantial performance improvement'
self.assertIn("rc:get_trading_calendar", rds.keys()[0])
def scrape_dividends(date: dt = None, timeout: int = 120) -> pd.DataFrame:
if date is None:
date = dt.now().replace(hour=0, minute=0, second=0, microsecond=0)
if get_trading_calendar(date, date).empty:
return pd.DataFrame()
day_of_month = str(date.day)
month = date.strftime('%B %Y')
driver = get_web_driver()
driver.get('https://www.thestreet.com/dividends/index.html')
table_x_path = '//*[@id="listed_divdates"]/table/tbody[2]'
if day_of_month != str(dt.now().day):
click_calendar(day_of_month, month, driver, timeout)
next_page = True
first_page = True
dividends_table = []
while next_page:
table = WebDriverWait(driver, timeout).until(
EC.visibility_of_element_located((By.XPATH, table_x_path)))
dividends_table += get_table_values(table)
next_page = click_next_page(table, first_page)
first_page = False
df = pd.DataFrame(
dividends_table,
columns=['symbol', 'company', 'amount', 'yield', 'exec_date'])
del df['yield']
df["amount"] = df['amount'].replace('[\$,]', '', regex=True).astype(float)
dividend_exdate = pd.to_datetime(df['exec_date'].iloc[0])
posix_dividend_exdate = datetime_to_posix(dividend_exdate)
df['exec_date'] = posix_dividend_exdate
df.drop_duplicates(inplace=True)
with engine.connect() as conn:
df.to_sql("dividends", conn, if_exists="append", index=False)
def get_order_expiration_status(order_id):
"""Before processing an order, we'll use logic to determine whether that order is still active. This function
return True if an order is expired, or false otherwise.
"""
with engine.connect() as conn:
time_in_force = conn.execute(
"SELECT time_in_force FROM orders WHERE id = %s;",
order_id).fetchone()[0]
if time_in_force == "until_cancelled":
return False
# posix_to_datetime
current_time = time.time()
with engine.connect() as conn:
time_placed = conn.execute(
"""SELECT timestamp
FROM order_status
WHERE order_id = %s
ORDER BY id LIMIT 0, 1;""",
order_id).fetchone()[0]
time_placed_nyc = posix_to_datetime(time_placed)
cal_ref_time = time_placed_nyc.date()
schedule = get_trading_calendar(cal_ref_time, cal_ref_time)
if schedule.empty:
next_day_schedule = get_next_trading_day_schedule(time_placed_nyc)
_, cutoff_time = get_schedule_start_and_end(next_day_schedule)
else:
if time_placed_nyc.hour >= END_OF_TRADE_HOUR:
next_day_schedule = get_next_trading_day_schedule(time_placed_nyc +
timedelta(
days=1))
_, cutoff_time = get_schedule_start_and_end(next_day_schedule)
else:
_, cutoff_time = get_schedule_start_and_end(schedule)
if current_time > cutoff_time:
return True
return False
def trade_time_index(timestamp_sr: pd.Series) -> List:
"""this function solves the problem of how to create a continuous, linear index across a bunch of purchases and
sales happening at different times across trade days. Simply trying to get the timestamp for a fixed number of bins
results in the algorithm creating bins for "non-event" times on weekend and between trading hours. This algorithm
create a "trade time index" that maps scalar time index values dependably to corresponding datetimes.
Note that the passed-in timestamp series must be sorted, meaning that the dataframe from the outer environment must
be sorted in orders for this to work.
"""
ls = timestamp_sr.to_list()
assert all(ls[i] <= ls[i + 1] for i in range(len(ls) - 1)) # enforces that timestamps are strictly sorted
start_time = timestamp_sr.min().date()
end_time = timestamp_sr.max().date()
trade_times_df = get_trading_calendar(start_time, end_time)
# when this happens it means that the game is young enough that we don't yet have any observations that occured
# during trading hours. In this case we won't worry about filtering our trading hours -- we'll just assign an index
# on the times available
if trade_times_df.empty or trade_times_df.iloc[-1]["market_close"] <= start_time:
return pd.cut(timestamp_sr, N_PLOT_POINTS, right=True, labels=False, include_lowest=False).to_list()
df = timestamp_sr.to_frame()
df["anchor_time"] = timestamp_sr.min()
df["time_diff"] = (df["timestamp"] - df["anchor_time"]).dt.total_seconds()
df.set_index("timestamp", inplace=True)
df.index = df.index.to_period("D")
del df["anchor_time"]
trade_times_df["last_close"] = trade_times_df["market_close"].shift(1)
trade_times_df["non_trading_seconds"] = (
trade_times_df["market_open"] - trade_times_df["last_close"]).dt.total_seconds().fillna(0)
trade_times_df["adjustment"] = trade_times_df["non_trading_seconds"].cumsum()
trade_times_df.set_index("market_open", inplace=True)
trade_times_df.index = trade_times_df.index.to_period("D")
adjustment_df = trade_times_df["adjustment"]
tt_df = df.join(adjustment_df)
tt_df["trade_time"] = tt_df["time_diff"] - tt_df["adjustment"]
return downsample_time_series(tt_df["trade_time"], N_PLOT_POINTS)
def make_stock_data_records():
with open("./database/fixtures/stock_data.json") as json_file:
stock_data = json.load(json_file)
timezone = pytz.timezone(TIMEZONE)
sample_days = list(set([x["date"] for x in stock_data["AMZN"]]))
sample_days.sort()
schedule = get_trading_calendar(start_date=dt.utcnow() -
timedelta(days=15),
end_date=dt.utcnow() + timedelta(days=15))
trading_days = []
for i in range(5):
trading_days.append(schedule.iloc[i]["market_open"])
ixic_value = ixic_value_0 = 10_473.83
gspc_value = gspc_value_0 = 3_215.57
dji_value = dji_value_0 = 27_734.71
price_records = []
index_records = []
for sample_day, simulated_day in zip(sample_days, trading_days):
for stock_symbol in stock_data.keys():
sample_day_subset = [
entry for entry in stock_data[stock_symbol]
if entry["date"] == sample_day
]
for record in sample_day_subset:
simulated_date = simulated_day.strftime("%Y-%m-%d")
str_time = f"{simulated_date} {record['minute']}"
base_date = pd.to_datetime(str_time, format="%Y-%m-%d %H:%M")
localized_date = timezone.localize(base_date)
posix_time = datetime_to_posix(localized_date)
price_records.append(
dict(symbol=stock_symbol,
price=record["average"],
timestamp=posix_time))
# add synthetic index data as well
ixic_value += np.random.normal(0, 0.0005 * ixic_value_0)
gspc_value += np.random.normal(0, 0.0005 * gspc_value_0)
dji_value += np.random.normal(0, 0.0005 * dji_value_0)
index_records.append(
dict(symbol="^IXIC",
value=ixic_value,
timestamp=posix_time))
index_records.append(
dict(symbol="^GSPC",
value=gspc_value,
timestamp=posix_time))
index_records.append(
dict(symbol="^DJI", value=dji_value, timestamp=posix_time))
# our simulation requires a full two weeks worth of data. append new entries for indexes and prices adding 7 days
# worth of time to each.
extended_price_records = deepcopy(price_records)
extended_index_records = deepcopy(index_records)
for price_record, index_record in zip(price_records, index_records):
pr_copy = deepcopy(price_record)
pr_copy["timestamp"] += SECONDS_IN_A_DAY * 7
extended_price_records.append(pr_copy)
ir_copy = deepcopy(index_record)
ir_copy["timestamp"] += SECONDS_IN_A_DAY * 7
extended_index_records.append(ir_copy)
return extended_price_records, extended_index_records
def get_day_start(start_time_dt: dt):
start_time = datetime_to_posix(start_time_dt)
schedule = get_trading_calendar(start_time_dt, start_time_dt)
if not schedule.empty:
start_time, _ = get_schedule_start_and_end(schedule)
return start_time
