class TestStockData(unittest.TestCase):
def setUp(self):
self.stocks = StockData()
def test_retrieve_stock_is_list(self):
assert type(self.stocks.stock_list) is list
def test_stock_list_has_data(self):
assert len(self.stocks.stock_list) > 5400
@unittest.skip("need to learn to mock things")
def test_retrieve_single_stock_is_dataframe(self):
history = self.stocks.get_history("AAPL")
assert type(history) is pd.DataFrame
@unittest.skip("need to learn to mock things")
def test_single_stock_has_data(self):
history = self.stocks.get_history("AAPL")
assert len(history) > 245
def test_set_histories_sets_variables(self):
self.stocks.set_histories(from_file=True)
prices, volumes = self.stocks.prices, self.stocks.volumes
assert type(prices) is pd.DataFrame
assert type(volumes) is pd.DataFrame
assert type(prices['AAPL']) is pd.Series
assert type(volumes['GOOG']) is pd.Series
def test_filter_by_volume(self):
self.stocks.set_histories(from_file=True)
self.stocks.filter_by_volume()
filtered = self.stocks.filter_by_volume(min=1000000)
assert len(self.stocks.prices.columns) > len(filtered.columns)
class TestStockData(unittest.TestCase):
def setUp(self):
self.stocks = StockData()
def test_retrieve_stock_is_list(self):
assert type(self.stocks.stock_list) is list
def test_stock_list_has_data(self):
assert len(self.stocks.stock_list) > 5400
@unittest.skip("need to learn to mock things")
def test_retrieve_single_stock_is_dataframe(self):
history = self.stocks.get_history("AAPL")
assert type(history) is pd.DataFrame
@unittest.skip("need to learn to mock things")
def test_single_stock_has_data(self):
history = self.stocks.get_history("AAPL")
assert len(history) > 245
def test_set_histories_sets_variables(self):
self.stocks.set_histories(from_file=True)
prices, volumes = self.stocks.prices, self.stocks.volumes
assert type(prices) is pd.DataFrame
assert type(volumes) is pd.DataFrame
assert type(prices['AAPL']) is pd.Series
assert type(volumes['GOOG']) is pd.Series
def test_filter_by_volume(self):
self.stocks.set_histories(from_file=True)
self.stocks.filter_by_volume()
filtered = self.stocks.filter_by_volume(min=1000000)
assert len(self.stocks.prices.columns) > len(filtered.columns)
def load_data(self):
"""
loads stock data .csv from inputted filepath string on the GUI
as StockData object, also autocompletes all inputs
using information provided by the csv.
Error handling
invalid filepath :
empty filepath or file could not be found.
invalid .csv :
.csv file is empty, missing date column, etc.
"""
filepath = Path(self.filePathEdit.text())
try:
self.stock_data = StockData(filepath)
start_date, end_date = self.stock_data.get_period()
period = f"{start_date} to {end_date}"
# auto-complete feauture
self.startDateEdit.setText(start_date)
self.endDateEdit.setText(end_date)
self.periodEdit.setText(period)
self.SMA1Edit.setText("15")
self.SMA2Edit.setText("50")
self.SMA1Checkbox.setChecked(False)
self.SMA2Checkbox.setChecked(False)
self.report(
f"Data loaded from {filepath}; period auto-selected: {start_date} to {end_date}."
)
print(self.stock_data.data)
except IOError as e:
self.report(
f"Filepath provided is invalid or fail to open .csv file. {e}")
except TypeError as e:
self.report(
f"The return tuple is probably (nan, nan) because .csv is empty"
)
def sync_data(stock_id):
stock_data = read(str(stock_id))
df_statement = _sync_statement(
stock_id, None if stock_data is None else stock_data.df_statement)
df_performance = _sync_performance(
stock_id, df_statement,
None if stock_data is None else stock_data.df_performance)
df_profit_matrix = _sync_profit_matrix(
stock_id, None if stock_data is None else stock_data.df_profit_matrix)
return StockData(
stock_id, df_performance, df_statement, df_profit_matrix
) if df_statement is not None and df_performance is not None and df_profit_matrix is not None else None
def stocks_api(stock_symbol):
# The api fetching stock data
url = 'https://financialmodelingprep.com/api/financials/'
api_income_statement = requests.get(url + 'income-statement/' +
stock_symbol)
api_balance_statement = requests.get(url + 'balance-sheet-statement/' +
stock_symbol)
api_cash_statement = requests.get(url + 'cash-flow-statement/' +
stock_symbol)
income_statement = response_processing(api_income_statement)
balance_statement = response_processing(api_balance_statement)
cash_statement = response_processing(api_cash_statement)
result = StockData(balance_statement, income_statement, cash_statement,
stock_symbol)
return result
def __find_stocks(self, html: str, institution: str = "") -> bool:
if html is None:
return False
soup = BeautifulSoup(html, 'html.parser')
tables = soup.find_all('div', id=B3StockParser.__STOCKS_TABLE_ID)
if tables is None:
return False
for stocks_table in tables:
try:
rows = stocks_table.table.tbody.find_all('tr')
for row in rows:
self._stocks_table.append(StockData(row, institution))
except AttributeError:
return False
return True
def test_lower_price_targets_is_series(self): sd = StockData() sd.set_histories(from_file=True) p = Predict(sd.prices) lower_targets = p.find_lower_targets(-30,.75,.85) assert type(lower_targets) is pd.Series
def test_raise_price_targets_is_series(self): sd = StockData() sd.set_histories(from_file=True) p = Predict(sd.prices) raise_targets = p.find_raise_targets(-10,1.15,1.25) assert type(raise_targets) is pd.Series
def test_lower_price_targets_is_series(self):
sd = StockData()
sd.set_histories(from_file=True)
p = Predict(sd.prices)
lower_targets = p.find_lower_targets(-30, .75, .85)
assert type(lower_targets) is pd.Series
def test_raise_price_targets_is_series(self):
sd = StockData()
sd.set_histories(from_file=True)
p = Predict(sd.prices)
raise_targets = p.find_raise_targets(-10, 1.15, 1.25)
assert type(raise_targets) is pd.Series
def get_evaluate_performance(stock_id, since_year, to_year=None):
params = normalize_params(stock_id, since_year, to_year)
print(params)
if params is None:
return None
stock_id = params.get('stock_id')
evaluate_since_year = params.get('since_year')
evaluate_to_year = params.get('to_year')
get_recent_four_seasons = params.get('get_resent_four_seasons')
dividend_policy_processor = DividendPolicyProcessor(stock_id)
df_dividend_policy = dividend_policy_processor.get_data_frames(
{'year': evaluate_to_year}, {'year': evaluate_to_year + 1})
try:
should_modify_since_year = False if df_dividend_policy is not None and str(
evaluate_since_year) in df_dividend_policy.index else True
except KeyError:
should_modify_since_year = True
print('get error')
evaluate_since_year = evaluate_since_year - 1 if should_modify_since_year and evaluate_since_year >= evaluate_to_year - 1 else evaluate_since_year
df_dividend_policy = dividend_policy_processor.get_data_frames(
{'year': evaluate_since_year}, {'year': evaluate_to_year + 1})
df_dividend_policy.sort_index(inplace=True)
df_income_statement = SimpleIncomeStatementProcessor().get_data_frames(
stock_id, {'year': evaluate_since_year - 1},
{'year': evaluate_to_year})
df_income_statement.sort_index(inplace=True)
balance_sheet_processor = SimpleBalanceSheetProcessor(stock_id)
df_balance_sheet = balance_sheet_processor.get_data_frames(
{'year': evaluate_since_year - 1}, {'year': evaluate_to_year})
price_measurement_processor = PriceMeasurementProcessor(stock_id)
df_prices = price_measurement_processor.get_data_frame()
results_data = []
results_index = []
for year in range(evaluate_since_year, evaluate_to_year + 1):
count_of_effects = len(
df_income_statement.loc["{}Q{}".format(year - 1, 1):"{}Q{}".
format(year, 4)].index)
if count_of_effects == 8:
sum_of_eps_two_years = \
df_income_statement.loc["{}Q{}".format(year - 1, 1):"{}Q{}".format(year, 4)].sum().loc["EPS"]
value_in_two_years = df_balance_sheet.loc[
"{}Q{}".format(year - 1, 1),
"每股淨值"] + df_balance_sheet.loc["{}Q{}".format(year, 4), "每股淨值"]
else:
sum_of_eps_two_years = df_income_statement.iloc[
-8:].loc[:, "EPS"].sum()
value_in_two_years = df_balance_sheet.iloc[
[-8, -1], ].loc[:, "每股淨值"].sum()
has_dividend = str(year) in df_dividend_policy.index
predict = not count_of_effects == 8 or not has_dividend
if has_dividend:
dr_in_two_years = df_dividend_policy.loc[str(year - 1):str(year),
"現金股利"].sum()
print('dr_in_two_years in ', year, ' = ', dr_in_two_years)
dividend_ratio_two_years = dr_in_two_years / sum_of_eps_two_years
else:
if len(results_data) > 0:
dividend_ratio_two_years = results_data[-1].get('兩年現金股利發放率')
else:
dividend_ratio_two_years = 0
print('year ', year, '兩年現金股利發放率 = ', dividend_ratio_two_years,
' EPS = ', df_income_statement.iloc[-4:].loc[:, "EPS"].sum())
dr_in_two_years = dividend_ratio_two_years * df_income_statement.iloc[
-4:].loc[:, "EPS"].sum()
# print('兩年現金股利發放率 in ', year, ':', dividend_ratio_two_years)
avg_roe_two_years = sum_of_eps_two_years / value_in_two_years
growth_ratio_of_eps = avg_roe_two_years * (1 -
dividend_ratio_two_years)
try:
dividend_yield = dr_in_two_years / 2 / float(
df_prices.loc[str(year), '平均股價'])
except Exception as e:
print('get', e, ' when get price for ', stock_id, ' in ', year)
dividend_yield = float(0)
results_index.append(str(year))
results_data.append({
'兩年現金股利發放率':
dividend_ratio_two_years,
'兩年平均ROE':
avg_roe_two_years,
'保留盈餘成長率':
growth_ratio_of_eps,
'現金股利殖利率':
dividend_yield,
'高登模型預期報酬率':
growth_ratio_of_eps + dividend_yield,
'預估值':
predict,
'現金股利':
df_dividend_policy.loc[str(year),
"現金股利"] if has_dividend else dr_in_two_years
})
merged_statement = pd.concat([df_income_statement, df_balance_sheet],
axis=1,
sort=False)
# print(df_results)
return StockData(stock_id, pd.DataFrame(results_data, index=results_index),
merged_statement,
get_matrix_level(stock_id, since_year, to_year))
def setUp(self):
self.stocks = StockData()
def setUp(self): self.stocks = StockData()
from datetime import datetime, timedelta from connect import DBConnection from stock_data import StockData, Predict if __name__ == '__main__': est = (datetime.utcnow() + timedelta(hours=-4)).timetuple() conn = DBConnection() count_untweeted = conn.get_num_untweeted_stocks() if est[6] == 6 or count_untweeted == 0: sd = StockData() sd.set_histories(from_file=False,num=5000) p = Predict(sd.prices) raise_targets = p.find_raise_targets(-10,1.15,1.25) lower_targets = p.find_lower_targets(-30,.75,.85) p.save_targets(raise_targets, True) p.save_targets(lower_targets, False)
def __init__(self, ticker_index, trend_period_days=21):
self.ticker_index = ticker_index
self.stock_data = StockData(ticker_index)
self.close = self.stock_data.close
self.trend_period = trend_period_days
self.ema_trend = self.compute_ema()
def deal_trade_data(self):
stock_data = StockData()
# 获取历史数据
stock_data.history_data()
# 获取实时数据
stock_data.real_time_data_all_stock()
def save_data_as_h5_file_(self, industry_name, hdf5_file_name):
industry_idxs = self.stock_idxs[industry_name]
for idx in industry_idxs:
stock_data_obj = StockData(idx)
stock_data = stock_data_obj.get_stocks_data()
self.save_stock_data_to_hdf5(stock_data, hdf5_file_name, idx)
class Main(qtw.QWidget, Ui_Form):
"""
handles user interaction, loads data and updates GUI
Attributes
.figure : Figure
matplotlib Figure object to contain the Axes object(s) and the FigureCanvas object
.ax : Axes
matplotlib Axes object is the 'plot' itself, the region of the image which contains the data
.canvas : FigureCanvas
matplotlib FigureCanvas is the area onto which the figure is drawn
.toolbar : NavigationToolbar
matplotlib NavigationToolbar is the UI that users can use to interact with the drawn plot
.canvasLayout : QVBoxLayout
PyQt5's object used to demark the location and contain the FigureCanvas and NavigationToolbar
.loadCSVButton : QPushButton
PyQt5's object that user can push to activate the load_data() function
.updateWindowButton : QPushButton
PyQt5's object that user can push to activate the update_canvas() function
.SMA1Checkbox : QCheckBox
PyQt5's object that user can tick to specify whether to include SMA1 plot in the canvas
.SMA2Checkbox : QCheckBox
PyQt5's object that user can tick to specify whether to include SMA2 plot in the canvas
.scrollWidget : QWidget
PyQt5's base object for that receives user inputs, as such it is interactable on the screen
.scrollLayout : QVBoxLayout
PyQt5's object used to demark the location and contain the QScrollArea
.scrollArea : QScrollArea
PyQt5's object's scrollable area on which status reports of the GUI actions are written
.filePathEdit : QLineEdit
PyQt5's object used to create a box into which user can input filepath (e.g. ../data/GOOG.csv)
.startDateEdit : QLineEdit
PyQt5's object used to create a box into which user can input start date (YYYY-MM-DD)
.endDateEdit : QLineEdit
PyQt5's object used to create a box into which user can input end date (YYYY-MM-DD)
.SMA1Edit : QLineEdit
PyQt5's object used to create a box into which user can input SMA1 window value (e.g. 15)
.SMA2Edit : QLineEdit
PyQt5's object used to create a box into which user can input SMA2 window value (e.g. 50)
.periodEdit : QLineEdit
PyQt5's object used to create a box which user can use to see the period used for the graph
"""
def __init__(self):
"""
initializes and sets up GUI widgets and its connections
"""
super().__init__()
self.setupUi(self)
self.setWindowTitle("Stock Chart & Moving Average Application")
# sets up figure to plot on, instantiates canvas and toolbar
self.figure, self.ax = plt.subplots()
self.canvas = FigureCanvas(self.figure)
self.toolbar = NavigationToolbar(self.canvas, self)
# attaches the toolbar and canvas to the canvas layout
self.canvasLayout.addWidget(self.toolbar)
self.canvasLayout.addWidget(self.canvas)
# sets up a scroll area to display GUI statuses
self.scrollWidget = qtw.QWidget()
self.scrollLayout = qtw.QVBoxLayout()
self.scrollWidget.setLayout(self.scrollLayout)
self.scrollArea.setWidget(self.scrollWidget)
# button & checkbox connections
self.loadCSVButton.clicked.connect(self.load_data)
self.updateWindowButton.clicked.connect(self.update_canvas)
self.SMA1Checkbox.stateChanged.connect(self.update_canvas)
self.SMA2Checkbox.stateChanged.connect(self.update_canvas)
# auto-complete feauture
self.filePathEdit.setText("../data/GOOG.csv")
def load_data(self):
"""
loads stock data .csv from inputted filepath string on the GUI
as StockData object, also autocompletes all inputs
using information provided by the csv.
Error handling
invalid filepath :
empty filepath or file could not be found.
invalid .csv :
.csv file is empty, missing date column, etc.
"""
filepath = Path(self.filePathEdit.text())
try:
self.stock_data = StockData(filepath)
start_date, end_date = self.stock_data.get_period()
period = f"{start_date} to {end_date}"
# auto-complete feauture
self.startDateEdit.setText(start_date)
self.endDateEdit.setText(end_date)
self.periodEdit.setText(period)
self.SMA1Edit.setText("15")
self.SMA2Edit.setText("50")
self.SMA1Checkbox.setChecked(False)
self.SMA2Checkbox.setChecked(False)
self.report(
f"Data loaded from {filepath}; period auto-selected: {start_date} to {end_date}."
)
print(self.stock_data.data)
except IOError as e:
self.report(
f"Filepath provided is invalid or fail to open .csv file. {e}")
except TypeError as e:
self.report(
f"The return tuple is probably (nan, nan) because .csv is empty"
)
def update_canvas(self):
"""
creates a datetime object from the inputted date string
of format YYYY-MM-DD. uses it to slice a copy of loaded
stock_data to be used to update graphics. checks
checkboxes first to see if SMA1, SMA2, Buy and Sell plots
need to be drawn. finally, updates graphic accordingly.
Error handling
invalid date format:
date format inside the .csv file is not YYYY-MM-DD
non-existent stock_data :
the selected range results in an empty dataframe
or end date < start date
non-existent data point :
data of that date does not exist,
or maybe because it is Out-Of-Bound
raised exceptions :
SMA1 and SMA2 values are the same,
or other exceptions raised
"""
self.date_format = '%Y-%m-%d'
try:
start_date = str(
datetime.strptime(self.startDateEdit.text(),
self.date_format).date())
end_date = str(
datetime.strptime(self.endDateEdit.text(),
self.date_format).date())
period = f"{start_date} to {end_date}"
self.periodEdit.setText(period)
# builds a list of graphs to plot by checking the tickboxes
column_headers = ['Close']
formats = ['k-']
if self.SMA1Checkbox.isChecked():
self.stock_data._calculate_SMA(int(self.SMA1Edit.text()))
column_headers.append(f"SMA{self.SMA1Edit.text()}")
formats.append('b-')
if self.SMA2Checkbox.isChecked():
self.stock_data._calculate_SMA(int(self.SMA2Edit.text()))
column_headers.append(f"SMA{self.SMA2Edit.text()}")
formats.append('m-')
if len(column_headers) == 3:
self.stock_data._calculate_crossover(column_headers[1],
column_headers[2],
column_headers[1])
column_headers.append('Sell')
formats.append('rv')
column_headers.append('Buy')
formats.append('g^')
self.selected_stock_data = self.stock_data.get_data(
start_date, end_date)
self.plot_graph(column_headers, formats)
self.report(
f"Plotting {column_headers} data from period: {start_date} to {end_date}."
)
print(self.selected_stock_data)
except ValueError as e:
self.report(
f"Time period has not been specified or does not match YYYY-MM-DD format, {e}."
)
except AssertionError as e:
self.report(f"Selected range is empty, {e}")
except KeyError as e:
self.report(f"Data for this date does not exist: {e}")
except Exception as e:
self.report(f"Exception encountered: {e}")
def plot_graph(self, column_headers, formats):
"""
plots graphs specified under column_headers using the formats
Parameters
column_headers : [str, str, ...]
a list containing column header names with data to be plotted
formats : [str, str, ...]
a list of matplotlib built-in style strings to indicate
whether to plot line or scatterplot and the colours
corresponding to each value in col_headers
(hence, must be same length)
Error handling
empty dataframe :
selected dataframe is empty
"""
self.ax.clear()
assert not self.selected_stock_data.empty
# matplotlib has its own internal representation of datetime
# date2num converts datetime.datetime to this internal representation
x_data = list(
mdates.date2num([
datetime.strptime(dates, self.date_format).date()
for dates in self.selected_stock_data.index.values
]))
for i in range(len(column_headers)):
if column_headers[i] in self.selected_stock_data.columns:
y_data = list(self.selected_stock_data[column_headers[i]])
self.ax.plot(x_data,
y_data,
formats[i],
label=column_headers[i])
self.report(f"{column_headers[i]} data is being plotted.")
else:
self.report(f"{column_headers[i]} data does not exist.")
# formatting
months_locator = mdates.MonthLocator()
months_format = mdates.DateFormatter('%b %Y')
self.ax.xaxis.set_major_locator(months_locator)
self.ax.xaxis.set_major_formatter(months_format)
self.ax.format_xdata = mdates.DateFormatter(self.date_format)
self.ax.format_ydata = lambda y: '$%1.2f' % y
self.ax.grid(True)
self.figure.autofmt_xdate()
self.figure.legend()
self.figure.tight_layout()
self.canvas.draw()
def report(self, string):
"""
given a report (string), update the scroll area with this report
Parameters
string : str
string of the report, usually the error message itself.
"""
report_text = qtw.QLabel(string)
self.scrollLayout.addWidget(report_text)
print(string)
def center(self):
"""
centers the fixed main window size according to user screen size
"""
screen = qtw.QDesktopWidget().screenGeometry()
main_window = self.geometry()
x = (screen.width() - main_window.width()) / 2
# pulls the window up slightly (arbitrary)
y = (screen.height() - main_window.height()) / 2 - 50
self.setFixedSize(main_window.width(), main_window.height())
self.move(x, y)