def predict_price_change():
'''Train model and predict probability of price change in the next day for each symbol.
If run on the non-trading day, e.g. weekends, the current probabilities are updated
'''
logger.info('Called')
symbols = db_init['symbols']
line = get_last_line(_prob_pred_file).split(',')
# If on the non-trading day, update the probability predictions
if line[0] == 'NA':
logger.info('Update the current predicted probabilities')
pop_last_line(_prob_pred_file)
prob_pred = {}
for symbol in symbols:
model = Model(symbol)
model.load_data()
model.fit()
neg, neu, pos = model.predict_proba()
prob_pred[symbol] = tuple(map(lambda x: round(x, 4), (neg, neu, pos)))
# Log probability predictions for the next trading day
prob = ['NA'] + [v for symbol in symbols for v in prob_pred[symbol]]
with open(_prob_pred_file, 'a') as f:
f.write(','.join(map(str, prob)) + '\n')
def get_curr_holding():
'''Get current holding'''
symbols = db_init['symbols']
line = get_last_line(_portfolio_file).split(',')
share = {s: int(v) for s, v in zip(symbols, line[1:-3:3])}
cash, total_value = float(line[-3]), float(line[-2])
return line[0], share, cash, total_value
def update_ta_feature(symbol: str):
'''Update ta features for the specified symbol with the latest daily time series data'''
logger.info('Called for %s', symbol)
db = db_init['db']
last_trade_day = Database.get_last_trade_day(db, symbol)
last_ta_line = get_last_line(_data_folder + '{0}_ta.csv'.format(symbol))
last_ta_day = last_ta_line.split(',')[0]
logger.info('Last trade day: %s, last ta day: %s', last_trade_day, last_ta_day)
if last_ta_day < last_trade_day:
data = Database.get_data_daily(db, symbol, start=_since_this_date)
build_ta_feature(symbol, data)
def update_news_feature(symbol: str):
'''Update mews features for the specified symbol with the latest news data.
Should be used on a daily basis
'''
logger.info('Called for %s', symbol)
db, symbols = db_init['db'], db_init['symbols']
if symbol == 'Market':
last_trade_day = Database.get_last_trade_day(db, symbols[0])
else:
last_trade_day = Database.get_last_trade_day(db, symbol)
# Get news data and build feature for the last trade day
news = Database.get_data_news(db, symbol, start=last_trade_day)
news_feature = build_news_feature(last_trade_day, news)
from_date, to_date = (news[0][0], news[-1][0]) if len(news) > 0 else ('-', '-')
logger.info('Get %d news, from %s to %s', len(news), from_date, to_date)
# Get the date for the last news feature in file to determine whether the
# newly built news feature should override the existing one or simply append
new_feature_file = _data_folder + '{0}_news.csv'.format(symbol)
last_news_line = get_last_line(new_feature_file)
last_news_day = last_news_line.split(',')[0]
logger.info('Last trade day: %s, last news day: %s', last_trade_day, last_news_day)
if last_news_day > last_trade_day:
logger.error('Unexpected behavior for last trade and news date')
if last_news_day == last_trade_day:
logger.info('Override the news feature for %s', last_news_day)
pop_last_line(new_feature_file)
else:
logger.info('Add the news feature for %s', last_trade_day)
with open(new_feature_file, 'a') as f:
f.write(','.join(map(str, news_feature)) + '\n')
log_info("Retrieving the price data (country code: %s)" % cc)
rest_apps = list(apps)
while len(rest_apps) != 0:
log_info("There are %d apps left" % len(rest_apps))
# Retrieve the first {APPS_PER_REQUEST} apps
this_time_apps = []
next_time_apps = []
for (app_id, app_name) in rest_apps:
if len(this_time_apps) < APPS_PER_REQUEST:
# Check if we have the price data for this app today
file_path = cc_dir + "/app_price_" + str(app_id) + ".txt"
if os.path.exists(file_path):
last_line = get_last_line(file_path)
time_str = last_line.split(' ')[0]
if time_to_day(int(time_str)) == time_to_day(START_TIME):
continue
this_time_apps.append((app_id, app_name))
else:
next_time_apps.append((app_id, app_name))
rest_apps = next_time_apps
# Retrieve the price of the first 100 apps
url = APP_URL_PREFIX
for (app_id, _) in this_time_apps:
url += str(app_id) + ','
def finalize_transaction():
'''Finalized the transaction for today based on the predicted probabilities and
update the fund using the lastest daily price data.
Rule to follow:
If today is a trading day, then buy shares at the open price, based on
the predicted probabilities and the derived trading strategy. The total
value at the end of day is calculated with the close price.
If not a trading day, do nothing.
'''
logger.info('Called')
db, symbols = db_init['db'], db_init['symbols']
# Probabilities of price change for this day
curr_prob = pop_last_line(_prob_pred_file).split(',')
if curr_prob[0] != 'NA':
logger.error('Probability data not available: %s', curr_prob[0])
last_transaction = get_last_line(_portfolio_file).split(',')
cash = float(last_transaction[-3])
last_trade_day = Database.get_last_trade_day(db, symbols[0])
logger.info('Last trade day: %s', last_trade_day)
logger.info('Last transaction date: %s', last_transaction[0])
if last_trade_day < last_transaction[0]:
logger.error('Unexpected behavior for last trade and transaction date')
# New daily price data is available to finalize the transaction
if last_trade_day > last_transaction[0] and curr_prob[0] == 'NA':
logger.info('Finalize transaction for: %s', last_trade_day)
init_share, init_cash, _ = get_init_holding()
curr_share = {
s: int(v)
for s, v in zip(symbols, last_transaction[1:-3:3])
}
probs = {
s: tuple(map(float, curr_prob[i * 3 + 1:i * 3 + 4]))
for i, s in enumerate(symbols)
}
curr_open, curr_close = {}, {}
# Get open and close price for this day
for symbol in symbols:
data = Database.get_data_daily(db=db,
symbol=symbol,
start=last_trade_day)[0]
curr_open[symbol], curr_close[symbol] = float(data[1]), float(
data[4])
# Buy/sell shares at the open price
for symbol in symbols:
# A simple strategy
pos, neg = probs[symbol][-1], probs[symbol][0]
transaction = int(round(init_share[symbol] * (pos - neg) / 2, 0))
# Do not consider short-selling
if curr_share[symbol] + transaction < 0:
transaction = -curr_share[symbol]
logger.info('Trading decision for %s (%d): %d share (%.4f, %.4f)',
symbol, curr_share[symbol], transaction, pos, neg)
# Log this transaction to trade history
with open(_trade_history_file, 'a') as f:
record = (last_trade_day, symbol, curr_share[symbol],
transaction, round(curr_open[symbol], 4))
f.write(','.join(map(str, record)) + '\n')
# Update number of shares and cash
curr_share[symbol] += transaction
cash -= transaction * curr_open[symbol]
# Total value and benchmark at the end of day
total = cash + sum(curr_share[symbol] * curr_close[symbol]
for symbol in symbols)
benchmark = init_cash + sum(init_share[symbol] * curr_close[symbol]
for symbol in symbols)
# Update the transaction information
curr_transaction = [last_trade_day]
for symbol in symbols:
curr_transaction += [
curr_share[symbol], curr_open[symbol], curr_close[symbol]
]
curr_transaction += [
round(cash, 4),
round(total, 4),
round(benchmark, 4)
]
with open(_portfolio_file, 'a') as f:
f.write(','.join(map(str, curr_transaction)) + '\n')
# Update the date for the probability predictions made for this day
curr_prob[0] = last_trade_day
_validate_cash()
with open(_prob_pred_file, 'a') as f:
f.write(','.join(curr_prob) + '\n')