def setup_db_for_hist_prices_storage(self, stock_sym_list):
""" Get the price and dividend history and store them to the database for the specified stock sym list.
The length of time depends on the date_interval specified.
Connection to database is assuemd to be set.
For one time large dataset (where the hist data is very large)
Args:
stock_sym_list (list): list of stock symbol.
"""
## set the class for extraction
histdata_extract = YFHistDataExtr()
histdata_extract.set_interval_to_retrieve(360*5)# assume for 5 years information
histdata_extract.enable_save_raw_file = 0
for sub_list in self.break_list_to_sub_list(stock_sym_list):
print ('processing sub list', sub_list)
histdata_extract.set_multiple_stock_list(sub_list)
histdata_extract.get_hist_data_of_all_target_stocks()
histdata_extract.removed_zero_vol_fr_dataset()
## save to one particular funciton
#save to sql -- hist table
histdata_extract.processed_data_df.to_sql(self.hist_data_tablename, self.con, flavor='sqlite',
schema=None, if_exists='append', index=True,
index_label=None, chunksize=None, dtype=None)
#save to sql -- div table
histdata_extract.all_stock_div_hist_df.to_sql(self.divdnt_data_tablename, self.con, flavor='sqlite',
schema=None, if_exists='append', index=True,
index_label=None, chunksize=None, dtype=None)
self.close_db()
def setup_db_for_hist_prices_storage(self,stock_sym_list,interval):
#get price history
histdata_extract = YFHistDataExtr()
histdata_extract.set_interval_to_retrieve(360*interval)
histdata_extract.enable_save_raw_file=0
for sub_list in self.break_list_to_sub_list(stock_sym_list):
print ('processing sub list', sub_list)
histdata_extract.set_multiple_stock_list(sub_list)
histdata_extract.get_hist_data_of_all_target_stocks()
histdata_extract.removed_zero_vol_fr_dataset()
# save to SQL table
histdata_extract.processed_data_df.to_sql(self.hist_data_tablename, self.con, flavor='sqlite',
schema=None, if_exists='append', index=True,
index_label=None, chunksize=None, dtype=None)
self.close_db()
def setup_db_for_hist_prices_storage(self, stock_sym_list):
""" Get the price and dividend history and store them to the database for the specified stock sym list.
The length of time depends on the date_interval specified.
Connection to database is assuemd to be set.
For one time large dataset (where the hist data is very large)
Args:
stock_sym_list (list): list of stock symbol.
"""
## set the class for extraction
histdata_extract = YFHistDataExtr()
histdata_extract.set_interval_to_retrieve(
360 * 5) # assume for 5 years information
histdata_extract.enable_save_raw_file = 0
for sub_list in self.break_list_to_sub_list(stock_sym_list):
print('processing sub list', sub_list)
histdata_extract.set_multiple_stock_list(sub_list)
histdata_extract.get_hist_data_of_all_target_stocks()
histdata_extract.removed_zero_vol_fr_dataset()
## save to one particular funciton
#save to sql -- hist table
histdata_extract.processed_data_df.to_sql(self.hist_data_tablename,
self.con,
flavor='sqlite',
schema=None,
if_exists='append',
index=True,
index_label=None,
chunksize=None,
dtype=None)
#save to sql -- div table
histdata_extract.all_stock_div_hist_df.to_sql(
self.divdnt_data_tablename,
self.con,
flavor='sqlite',
schema=None,
if_exists='append',
index=True,
index_label=None,
chunksize=None,
dtype=None)
self.close_db()
def calculate_bands(ticker_symbol):
# Delete the old bands image for this picture
os.system("rm ./static/pictures/%s.png" % ticker_symbol)
data_ext = YFHistDataExtr()
data_ext.set_interval_to_retrieve(400) #in days
data_ext.set_multiple_stock_list([str(ticker_symbol)])
data_ext.get_hist_data_of_all_target_stocks()
# convert the date column to date object
data_ext.all_stock_df['Date'] = pandas.to_datetime(
data_ext.all_stock_df['Date'])
temp_data_set = data_ext.all_stock_df.sort(
'Date', ascending=True) #sort to calculate the rolling mean
temp_data_set['20d_ma'] = pandas.rolling_mean(temp_data_set['Adj Close'],
window=5)
temp_data_set['50d_ma'] = pandas.rolling_mean(temp_data_set['Adj Close'],
window=50)
temp_data_set['Bol_upper'] = pandas.rolling_mean(
temp_data_set['Adj Close'], window=80) + 2 * pandas.rolling_std(
temp_data_set['Adj Close'], 80, min_periods=80)
temp_data_set['Bol_lower'] = pandas.rolling_mean(
temp_data_set['Adj Close'], window=80) - 2 * pandas.rolling_std(
temp_data_set['Adj Close'], 80, min_periods=80)
temp_data_set['Bol_BW'] = (
(temp_data_set['Bol_upper'] - temp_data_set['Bol_lower']) /
temp_data_set['20d_ma']) * 100
temp_data_set['Bol_BW_200MA'] = pandas.rolling_mean(
temp_data_set['Bol_BW'], window=50) #cant get the 200 daa
temp_data_set['Bol_BW_200MA'] = temp_data_set['Bol_BW_200MA'].fillna(
method='backfill') ##?? ,may not be good
temp_data_set['20d_exma'] = pandas.ewma(temp_data_set['Adj Close'],
span=20)
temp_data_set['50d_exma'] = pandas.ewma(temp_data_set['Adj Close'],
span=50)
data_ext.all_stock_df = temp_data_set.sort(
'Date', ascending=False) #revese back to original
data_ext.all_stock_df.plot(
x='Date',
y=['Adj Close', '20d_ma', '50d_ma', 'Bol_upper', 'Bol_lower'])
#data_ext.all_stock_df.plot(x='Date', y=['Bol_BW','Bol_BW_200MA' ])
plt.savefig('./static/pictures/%s.png' % ticker_symbol)
return temp_data_set['Adj Close'], temp_data_set[
'Bol_lower'], temp_data_set['20d_ma']
def setup_db_for_hist_prices_storage(self, stock_sym_list, interval):
#get price history
histdata_extract = YFHistDataExtr()
histdata_extract.set_interval_to_retrieve(360 * interval)
histdata_extract.enable_save_raw_file = 0
for sub_list in self.break_list_to_sub_list(stock_sym_list):
print('processing sub list', sub_list)
histdata_extract.set_multiple_stock_list(sub_list)
histdata_extract.get_hist_data_of_all_target_stocks()
histdata_extract.removed_zero_vol_fr_dataset()
# save to SQL table
histdata_extract.processed_data_df.to_sql(self.hist_data_tablename,
self.con,
flavor='sqlite',
schema=None,
if_exists='append',
index=True,
index_label=None,
chunksize=None,
dtype=None)
self.close_db()
ss.get_all_criteria_fr_file()
ss.process_criteria()
print "stocks left after basic filtering: ", len(ss.modified_df)
ss.modified_df.to_csv(final_store_filename, index = False)
print "Basic Filtering of stock data -- Done. \n"
if 'c_backup' in partial_run:
## testing, to remove after this
#ss.modified_df = ss.modified_df.head()
## 3 days trends data
print "Getting Trends data"
trend_ext = YFHistDataExtr()
trend_ext.set_interval_to_retrieve(365*5)
trend_ext.enable_save_raw_file = 0
trend_ext.set_multiple_stock_list(list(ss.modified_df['SYMBOL']))
trend_ext.run_all_hist_data()
full_stock_data_df = pandas.merge(ss.modified_df, trend_ext.all_stock_combined_post_data_df, on = 'SYMBOL', how ='left')
print "Getting Trends data -- Done. \n"
#full_stock_data_df.to_csv(r'c:\data\full_oct21.csv', index = False)
if 'c_pre' in partial_run:
""" Update the database """
print 'Updating the database with latest hist price.'
datastore = FinanceDataStore(db_full_path)
stock_list = list(ss.modified_df['SYMBOL'])
datastore.scan_and_input_recent_prices(stock_list,5)
print 'Updating the database with latest hist price -- Done\n'
if 'c' in partial_run:
