def test_time_handlers(self): posix_time = 1590165714.1528566 actual_date = dt(2020, 5, 22, 12, 41, 54, 152857) localizer = pytz.timezone(TIMEZONE) localized_date = localizer.localize(actual_date) self.assertEqual(posix_to_datetime(posix_time), localized_date) self.assertAlmostEqual(posix_time, datetime_to_posix(localized_date), 0) mexico_date = actual_date.replace(hour=11) localizer = pytz.timezone("America/Mexico_City") localized_date = localizer.localize(mexico_date) self.assertAlmostEqual(posix_time, datetime_to_posix(localized_date), 0) # Pre-stage all of the mocked current time values that will be called sequentially in the tests below. # ---------------------------------------------------------------------------------------------------- with patch('backend.logic.base.time') as current_time_mock: # Check during trading day just one second before and after open/close schedule = get_trading_calendar(actual_date, actual_date) start_day, end_day = [ datetime_to_posix(x) for x in schedule.iloc[0][["market_open", "market_close"]] ] current_time_mock.time.side_effect = [ posix_time, # during trading day posix_time + 8 * 60 * 60, # 8 hours later--after trading day 1608908400, # Christmas 2020, Friday, 11am in NYC. We want to verify that we're accounting for holidays start_day - 1, # one second before trading day (start_day + end_day) / 2, # right in the middle of trading day end_day + 1 # one second after trading day ] self.assertTrue(during_trading_day()) self.assertFalse(during_trading_day()) self.assertFalse(during_trading_day()) self.assertFalse(during_trading_day()) self.assertTrue(during_trading_day()) self.assertFalse(during_trading_day()) # Finally, just double-check that the real-time, default invocation works as expected posix_now = time.time() nyc_now = posix_to_datetime(posix_now) schedule = get_trading_calendar(nyc_now, nyc_now) during_trading = False if not schedule.empty: start_day, end_day = [ datetime_to_posix(x) for x in schedule.iloc[0][["market_open", "market_close"]] ] during_trading = start_day <= posix_now <= end_day # FYI: there is a non-zero chance that this test will fail at exactly the beginning or end of a trading day self.assertEqual(during_trading, during_trading_day())
def make_payout_table_entry(start_date: dt, end_date: dt, winner: str, payout: float, type: str, benchmark: str = None, score: float = None): if score is None: formatted_score = NA_TEXT_SYMBOL else: assert benchmark in ["return_ratio", "sharpe_ratio"] formatted_score = round(score, 3) return dict( Start=datetime_to_posix(start_date), End=datetime_to_posix(end_date), Winner=winner, Payout=payout, Type=type, Score=formatted_score )
def serialize_and_pack_order_performance_chart(df: pd.DataFrame, game_id: int, user_id: int): if df.empty: chart_json = make_null_chart("Waiting for orders...") else: apply_validation(df, order_performance_schema, True) plot_df = add_bookends(df, group_var="order_label", condition_var="fifo_balance") plot_df["cum_pl"] = plot_df.groupby("order_label")["realized_pl"].cumsum() plot_df["timestamp"] = plot_df["timestamp"].apply(lambda x: posix_to_datetime(x)) plot_df.set_index("timestamp", inplace=True) plot_df = plot_df.groupby("order_label", as_index=False).resample(f"{RESAMPLING_INTERVAL}T").last().ffill() plot_df = plot_df.reset_index(level=1) plot_df = filter_for_trade_time(plot_df) plot_df = append_price_data_to_balance_histories(plot_df) plot_df.sort_values(["order_label", "timestamp"], inplace=True) plot_df = add_time_labels(plot_df) plot_df = plot_df.groupby(["order_label", "t_index"], as_index=False).agg("last") plot_df["label"] = plot_df["timestamp"].apply(lambda x: datetime_to_posix(x)).astype(float) plot_df.sort_values("timestamp", inplace=True) plot_df["total_pl"] = plot_df["cum_pl"] + plot_df["fifo_balance"] * plot_df["price"] - ( 1 - plot_df["total_pct_sold"]) * plot_df["basis"] plot_df["return"] = 100 * plot_df["total_pl"] / plot_df["basis"] label_colors = assign_colors(plot_df["order_label"].unique()) plot_df["color"] = plot_df["order_label"].apply(lambda x: label_colors.get(x)) chart_json = make_chart_json(plot_df, "order_label", "return", "label", colors=plot_df["color"].unique()) s3_cache.set(f"{game_id}/{user_id}/{ORDER_PERF_CHART_PREFIX}", json.dumps(chart_json))
def add_time_labels(df: pd.DataFrame, time_col: dt = "timestamp") -> pd.DataFrame: df.sort_values(time_col, inplace=True) df["t_index"] = trade_time_index(df[time_col]) labels = df.groupby("t_index", as_index=False)[time_col].max().rename(columns={time_col: "label"}) df = df.merge(labels, how="inner", on="t_index") df["label"] = df["label"].apply(lambda x: datetime_to_posix(x)).astype(float) return df
def scrape_dividends(date: dt = None, timeout: int = 120) -> pd.DataFrame: if date is None: date = dt.now().replace(hour=0, minute=0, second=0, microsecond=0) if get_trading_calendar(date, date).empty: return pd.DataFrame() day_of_month = str(date.day) month = date.strftime('%B %Y') driver = get_web_driver() driver.get('https://www.thestreet.com/dividends/index.html') table_x_path = '//*[@id="listed_divdates"]/table/tbody[2]' if day_of_month != str(dt.now().day): click_calendar(day_of_month, month, driver, timeout) next_page = True first_page = True dividends_table = [] while next_page: table = WebDriverWait(driver, timeout).until( EC.visibility_of_element_located((By.XPATH, table_x_path))) dividends_table += get_table_values(table) next_page = click_next_page(table, first_page) first_page = False df = pd.DataFrame( dividends_table, columns=['symbol', 'company', 'amount', 'yield', 'exec_date']) del df['yield'] df["amount"] = df['amount'].replace('[\$,]', '', regex=True).astype(float) dividend_exdate = pd.to_datetime(df['exec_date'].iloc[0]) posix_dividend_exdate = datetime_to_posix(dividend_exdate) df['exec_date'] = posix_dividend_exdate df.drop_duplicates(inplace=True) with engine.connect() as conn: df.to_sql("dividends", conn, if_exists="append", index=False)
def make_null_chart(null_label: str): """Null chart function for when a game has just barely gotten going / has started after hours and there's no data. For now this function is a bit unnecessary, but the idea here is to be really explicit about what's happening so that we can add other attributes later if need be. """ schedule = get_next_trading_day_schedule(dt.utcnow()) start, end = [posix_to_datetime(x) for x in get_schedule_start_and_end(schedule)] labels = [datetime_to_posix(t) for t in pd.date_range(start, end, N_PLOT_POINTS)] data = [STARTING_VIRTUAL_CASH for _ in labels] return dict(labels=labels, datasets=[ dict(label=null_label, data=data, borderColor=NULL_RGBA, backgroundColor=NULL_RGBA, fill=False)])
def add_virtual_cash(game_id: int, user_id: int, dividend_id: int, amount: float): current_cash = get_current_game_cash_balance(user_id, game_id) + amount now = datetime_to_posix(dt.now()) add_row("game_balances", user_id=user_id, game_id=game_id, timestamp=now, balance_type='virtual_cash', balance=current_cash, dividend_id=dividend_id, transaction_type="stock_dividend")
def get_splits(start_time: float, end_time: float) -> pd.DataFrame: if start_time is None: start_time = datetime_to_posix(dt.utcnow().replace(hour=0, minute=0)) if end_time is None: end_time = time.time() # get active games and symbols with engine.connect() as conn: return pd.read_sql( "SELECT * FROM stock_splits WHERE exec_date >= %s AND exec_date <= %s", conn, params=[start_time, end_time])
def make_orders_per_active_user(): # get total number of active users on a given day (in at least one active game, or in a live single player mode) with engine.connect() as conn: game_status = pd.read_sql( """ SELECT game_id, users, timestamp FROM game_status WHERE status NOT IN ('expired', 'pending')""", conn) order_status = pd.read_sql( "SELECT id, timestamp FROM order_status WHERE status = 'pending';", conn) game_status["timestamp"] = game_status["timestamp"].apply( lambda x: posix_to_datetime(x)) game_status["users"] = game_status["users"].apply(lambda x: json.loads(x)) game_status["user_count"] = game_status["users"].apply(lambda x: len(x)) # get total number of active users per day complete_counts_array = [] for game_id in game_status["game_id"].unique(): game_subset = game_status[game_status["game_id"] == game_id] if game_subset.shape == 2: complete_counts_array += game_subset[["user_count", "timestamp" ]].to_dict(orient="records") continue user_count = game_subset.iloc[0]["user_count"] original_entry = dict(user_count=user_count, timestamp=game_subset.iloc[0]["timestamp"], game_id=game_id) bookend = dict(user_count=user_count, timestamp=posix_to_datetime(time.time()), game_id=game_id) complete_counts_array += [original_entry, bookend] user_count_df = pd.DataFrame(complete_counts_array) expanded_df = user_count_df.groupby("game_id").apply(expand_counts) expanded_df.index = expanded_df.index.droplevel(0) daily_active_users = expanded_df.resample("D").sum() order_status["timestamp"] = order_status["timestamp"].apply( lambda x: posix_to_datetime(x)) order_status.set_index("timestamp", inplace=True) order_totals = order_status.resample("D").count() df = pd.concat([daily_active_users, order_totals], axis=1) df["orders_per_users"] = df["id"] / df["user_count"] df.fillna(0, inplace=True) df = df.reset_index() df["timestamp"] = df["timestamp"].apply( lambda x: datetime_to_posix(x)).astype(float) return df
def serialize_and_pack_rankings(): cutoff_time = datetime_to_posix(dt.utcnow() - DateOffset(months=3)) with engine.connect() as conn: user_df = pd.read_sql(""" SELECT user_id, rating, username, profile_pic, n_games, total_return, basis, sr.timestamp FROM stockbets_rating sr INNER JOIN ( SELECT game_id, MAX(id) AS max_id FROM game_status WHERE status = 'finished' AND timestamp >= %s GROUP BY game_id ) gs ON gs.game_id = sr.game_id INNER JOIN ( SELECT id, username, profile_pic FROM users ) u ON u.id = sr.user_id """, conn, params=[cutoff_time]) indexes_df = pd.read_sql(""" SELECT user_id, rating, imd.username, imd.profile_pic, n_games, total_return, basis, sr.timestamp FROM stockbets_rating sr INNER JOIN ( SELECT game_id, MAX(id) AS max_id FROM game_status WHERE status = 'finished' AND timestamp >= %s GROUP BY game_id ) gs ON gs.game_id = sr.game_id INNER JOIN ( SELECT symbol, `name` AS username, avatar AS profile_pic FROM index_metadata ) imd ON sr.index_symbol = imd.symbol """, conn, params=[cutoff_time]) df = pd.concat([user_df, indexes_df]) df["basis_times_return"] = df["total_return"] * df["basis"] total_basis_df = df.groupby("username", as_index=False)["basis"].sum().rename(columns={"basis": "total_basis"}) df = df.merge(total_basis_df, on="username").sort_values(["username", "timestamp"]) stats_df = df.groupby("username", as_index=False).agg({"user_id": "first", "rating": "last", "profile_pic": "first", "n_games": "last", "basis_times_return": "sum", "total_basis": "first"}) stats_df["three_month_return"] = stats_df["basis_times_return"] / stats_df["total_basis"] del stats_df["basis_times_return"] del stats_df["total_basis"] stats_df.sort_values("rating", ascending=False, inplace=True) stats_df = stats_df.where(pd.notnull(stats_df), None) update_player_rank(stats_df) rds.set(PUBLIC_LEADERBOARD_PREFIX, json.dumps(stats_df.to_dict(orient="records")))
def test_apply_dividends_to_games(self): user_id = 1 date = datetime_to_posix(dt.now().replace(hour=0, minute=0, second=0, microsecond=0)) amzn_dividend = 10 tsla_dividend = 20 envidia_dividend = 15 fake_dividends = ([['AMZN', 'Amazon INC', amzn_dividend, date], ['TSLA', 'Tesla Motors', tsla_dividend, date], ['NVDA', 'Envidia SA', envidia_dividend, date]]) fake_dividends = pd.DataFrame( fake_dividends, columns=['symbol', 'company', 'amount', 'exec_date']) insert_dividends_to_db(fake_dividends) user_1_game_8_balance = get_current_game_cash_balance(user_id, 8) user_1_game_3_balance = get_current_game_cash_balance(user_id, 3) should_remain_1_000_000 = get_current_game_cash_balance(user_id, 6) apply_dividends_to_stocks() # user 1, game 8 is holding NVDA. we should see their holding * the dividend in their updated cash balance envidia_holding = get_current_stock_holding(user_id, 8, "NVDA") self.assertEqual( get_current_game_cash_balance(user_id, 8) - user_1_game_8_balance, envidia_holding * envidia_dividend) # user 1, game 3 is holding AMZN, TSLA, and NVDA. Their change in cash balance should be equal to the sum of # each position * its corresponding dividend active_balances = get_active_balances(3, user_id) merged_table = fake_dividends.merge(active_balances, how="left", on="symbol") merged_table[ "payout"] = merged_table["amount"] * merged_table["balance"] total_dividend = merged_table["payout"].sum() self.assertEqual( get_current_game_cash_balance(user_id, 3) - user_1_game_3_balance, total_dividend) # user 1 isn't holding any dividend-paying shares in game 6. they should have no cash balance change self.assertEqual(get_current_game_cash_balance(user_id, 6), should_remain_1_000_000)
def make_stock_data_records(): with open("./database/fixtures/stock_data.json") as json_file: stock_data = json.load(json_file) timezone = pytz.timezone(TIMEZONE) sample_days = list(set([x["date"] for x in stock_data["AMZN"]])) sample_days.sort() schedule = get_trading_calendar(start_date=dt.utcnow() - timedelta(days=15), end_date=dt.utcnow() + timedelta(days=15)) trading_days = [] for i in range(5): trading_days.append(schedule.iloc[i]["market_open"]) ixic_value = ixic_value_0 = 10_473.83 gspc_value = gspc_value_0 = 3_215.57 dji_value = dji_value_0 = 27_734.71 price_records = [] index_records = [] for sample_day, simulated_day in zip(sample_days, trading_days): for stock_symbol in stock_data.keys(): sample_day_subset = [ entry for entry in stock_data[stock_symbol] if entry["date"] == sample_day ] for record in sample_day_subset: simulated_date = simulated_day.strftime("%Y-%m-%d") str_time = f"{simulated_date} {record['minute']}" base_date = pd.to_datetime(str_time, format="%Y-%m-%d %H:%M") localized_date = timezone.localize(base_date) posix_time = datetime_to_posix(localized_date) price_records.append( dict(symbol=stock_symbol, price=record["average"], timestamp=posix_time)) # add synthetic index data as well ixic_value += np.random.normal(0, 0.0005 * ixic_value_0) gspc_value += np.random.normal(0, 0.0005 * gspc_value_0) dji_value += np.random.normal(0, 0.0005 * dji_value_0) index_records.append( dict(symbol="^IXIC", value=ixic_value, timestamp=posix_time)) index_records.append( dict(symbol="^GSPC", value=gspc_value, timestamp=posix_time)) index_records.append( dict(symbol="^DJI", value=dji_value, timestamp=posix_time)) # our simulation requires a full two weeks worth of data. append new entries for indexes and prices adding 7 days # worth of time to each. extended_price_records = deepcopy(price_records) extended_index_records = deepcopy(index_records) for price_record, index_record in zip(price_records, index_records): pr_copy = deepcopy(price_record) pr_copy["timestamp"] += SECONDS_IN_A_DAY * 7 extended_price_records.append(pr_copy) ir_copy = deepcopy(index_record) ir_copy["timestamp"] += SECONDS_IN_A_DAY * 7 extended_index_records.append(ir_copy) return extended_price_records, extended_index_records
def get_dividends_for_date(date: dt = None) -> pd.DataFrame: if date is None: date = dt.now().replace(hour=0, minute=0, second=0, microsecond=0) posix = datetime_to_posix(date) return pd.DataFrame( query_to_dict("SELECT * FROM dividends WHERE exec_date=%s", posix))
def get_day_start(start_time_dt: dt): start_time = datetime_to_posix(start_time_dt) schedule = get_trading_calendar(start_time_dt, start_time_dt) if not schedule.empty: start_time, _ = get_schedule_start_and_end(schedule) return start_time
def test_winner_payouts(self): """Use canonical game #3 to simulate a series of winner calculations on the test data. Note that since we only have a week of test data, we'll effectively recycle the same information via mocks """ game_id = 3 user_ids = get_active_game_user_ids(game_id) self.assertEqual(user_ids, [1, 3, 4]) game_info = get_game_info(game_id) n_players = len(user_ids) pot_size = n_players * game_info["buy_in"] self.assertEqual(pot_size, 300) last_payout_date = get_last_sidebet_payout(game_id) self.assertIsNone(last_payout_date) offset = make_date_offset(game_info["side_bets_period"]) self.assertEqual(offset, DateOffset(days=7)) start_time = game_info["start_time"] end_time = game_info["end_time"] self.assertEqual(start_time, simulation_start_time) n_sidebets = n_sidebets_in_game(start_time, end_time, offset) self.assertEqual(n_sidebets, 2) # we'll mock in daily portfolio values to speed up the time this test takes user_1_portfolio = portfolio_value_by_day(game_id, 1, start_time, end_time) user_3_portfolio = portfolio_value_by_day(game_id, 3, start_time, end_time) user_4_portfolio = portfolio_value_by_day(game_id, 4, start_time, end_time) # expected sidebet dates start_dt = posix_to_datetime(start_time) end_dt = posix_to_datetime(end_time) sidebet_dates = get_expected_sidebets_payout_dates( start_dt, end_dt, game_info["side_bets_perc"], offset) sidebet_dates_posix = [datetime_to_posix(x) for x in sidebet_dates] with patch("backend.logic.metrics.portfolio_value_by_day" ) as portfolio_mocks, patch( "backend.logic.base.time") as base_time_mock: time = Mock() time_1 = datetime_to_posix(posix_to_datetime(start_time) + offset) time_2 = end_time time.time.side_effect = base_time_mock.time.side_effect = [ time_1, time_2 ] portfolio_mocks.side_effect = [ user_1_portfolio, user_3_portfolio, user_4_portfolio ] * 4 winner_id, score = get_winner(game_id, start_time, end_time, game_info["benchmark"]) log_winners(game_id, time.time()) sidebet_entry = query_to_dict( "SELECT * FROM winners WHERE id = 1;")[0] self.assertEqual(sidebet_entry["winner_id"], winner_id) self.assertAlmostEqual(sidebet_entry["score"], score, 4) side_pot = pot_size * (game_info["side_bets_perc"] / 100) / n_sidebets self.assertEqual(sidebet_entry["payout"], side_pot * PERCENT_TO_USER) self.assertEqual(sidebet_entry["type"], "sidebet") self.assertEqual(sidebet_entry["start_time"], start_time) self.assertEqual(sidebet_entry["end_time"], sidebet_dates_posix[0]) self.assertEqual(sidebet_entry["timestamp"], time_1) log_winners(game_id, time.time()) sidebet_entry = query_to_dict( "SELECT * FROM winners WHERE id = 2;")[0] self.assertEqual(sidebet_entry["winner_id"], winner_id) self.assertAlmostEqual(sidebet_entry["score"], score, 4) self.assertEqual(sidebet_entry["payout"], side_pot * PERCENT_TO_USER) self.assertEqual(sidebet_entry["type"], "sidebet") self.assertEqual(sidebet_entry["start_time"], sidebet_dates_posix[0]) self.assertEqual(sidebet_entry["end_time"], sidebet_dates_posix[1]) self.assertEqual(sidebet_entry["timestamp"], time_2) overall_entry = query_to_dict( "SELECT * FROM winners WHERE id = 3;")[0] final_payout = pot_size * (1 - game_info["side_bets_perc"] / 100) self.assertEqual(overall_entry["payout"], final_payout * PERCENT_TO_USER) with self.engine.connect() as conn: df = pd.read_sql("SELECT * FROM winners", conn) self.assertEqual(df.shape, (3, 10)) serialize_and_pack_winners_table(game_id) payouts_table = s3_cache.unpack_s3_json(f"{game_id}/{PAYOUTS_PREFIX}") self.assertEqual(len(payouts_table["data"]), 3) self.assertTrue( sum([x["Type"] == "Sidebet" for x in payouts_table["data"]]), 2) self.assertTrue( sum([x["Type"] == "Overall" for x in payouts_table["data"]]), 1) for entry in payouts_table["data"]: if entry["Type"] == "Sidebet": self.assertEqual(entry["Payout"], side_pot * PERCENT_TO_USER) else: self.assertEqual(entry["Payout"], final_payout * PERCENT_TO_USER)
def log_winners(game_id: int, current_time: float): update_performed = False game_info = query_to_dict("SELECT * FROM games WHERE id = %s", game_id)[0] game_start, game_end, start_dt, end_dt, benchmark, side_bets_perc, stakes, offset = get_winners_meta_data(game_id) start_dt = posix_to_datetime(game_start) end_dt = posix_to_datetime(game_end) # If we have sidebets to monitor, see if we have a winner if side_bets_perc: last_interval_end = get_last_sidebet_payout(game_id) if not last_interval_end: last_interval_end = game_start last_interval_dt = posix_to_datetime(last_interval_end) payout_time = datetime_to_posix(last_interval_dt + offset) if check_if_payout_time(current_time, payout_time): win_type = "sidebet" curr_time_dt = posix_to_datetime(current_time) anchor_time = last_interval_dt + timedelta(days=1) expected_sidebet_dates = get_expected_sidebets_payout_dates(start_dt, end_dt, side_bets_perc, offset) curr_interval_end = [date for date in expected_sidebet_dates if anchor_time < date <= curr_time_dt][0] curr_interval_posix = datetime_to_posix(curr_interval_end) winner_id, score = get_winner(game_id, last_interval_end, curr_interval_posix, benchmark) payout = get_sidebet_payout(game_id, side_bets_perc, offset, stakes) winner_table_id = add_row("winners", game_id=game_id, winner_id=winner_id, score=float(score), timestamp=current_time, payout=payout, type=win_type, benchmark=benchmark, end_time=curr_interval_posix, start_time=last_interval_end) update_performed = True if stakes == "real": payment_profile = get_payment_profile_uuids([winner_id])[0] send_paypal_payment( uuids=[payment_profile["uuid"]], amount=payout, payment_type="sidebet", email_subject=f"Congrats on winning the {game_info['side_bets_period']} sidebet!", email_content=f"You came out on top in {game_info['title']} this week. Here's your payment of {USD_FORMAT.format(payout)}", note_content="Keep on crushing it." ) add_row("payments", user_id=winner_id, profile_id=payment_profile["id"], game_id=game_id, winner_table_id=winner_table_id, type=win_type, amount=payout, currency="USD", direction="outflow", timestamp=current_time) # if we've reached the end of our game, pay out the winner and mark the game as completed if current_time >= game_end: win_type = "overall" payout = get_overall_payout(game_id, side_bets_perc, stakes) winner_id, score = get_winner(game_id, game_start, game_end, benchmark) winner_table_id = add_row("winners", game_id=game_id, winner_id=winner_id, benchmark=benchmark, score=float(score), start_time=game_start, end_time=game_end, payout=payout, type=win_type, timestamp=current_time) update_performed = True if stakes == "real": pass # TODO: uncomment when we're able to take real money # payment_profile = get_payment_profile_uuids([winner_id])[0] # send_paypal_payment( # uuids=[payment_profile["uuid"]], # amount=payout, # payment_type="overall", # email_subject=f"Congrats on winning the {game_info['title']}!", # email_content=f"You were the overall winner of {game_info['title']}. Awesome work. Here's your payment of {USD_FORMAT.format(payout)}. Come back soon!", # note_content="Keep on crushing it." # ) # add_row("payments", user_id=winner_id, profile_id=payment_profile["id"], game_id=game_id, # winner_table_id=winner_table_id, type=win_type, amount=payout, currency="USD", # direction="outflow", timestamp=current_time) return update_performed