def scan_and_input_recent_prices(self,
stock_sym_list,
num_days_for_updates=10):
""" Another method to input the data to database. For shorter duration of the dates.
Function for storing the recent prices and set it to the databse.
Use with the YQL modules.
Args:
stock_sym_list (list): stock symbol list.
Kwargs:
num_days_for_updates: number of days to update. Cannot be set too large a date.
Default 10 days.
"""
w = YComDataExtr()
w.set_full_stocklist_to_retrieve(stock_sym_list)
w.set_hist_data_num_day_fr_current(num_days_for_updates)
w.get_all_hist_data()
## save to one particular funciton
#save to sql -- hist table
w.datatype_com_data_allstock_df.to_sql(self.hist_data_tablename,
self.con,
flavor='sqlite',
schema=None,
if_exists='append',
index=True,
index_label=None,
chunksize=None,
dtype=None)
def scan_and_input_recent_prices(self, stock_sym_list, num_days_for_updates=10):
""" Another method to input the data to database. For shorter duration of the dates.
Function for storing the recent prices and set it to the databse.
Use with the YQL modules.
Args:
stock_sym_list (list): stock symbol list.
Kwargs:
num_days_for_updates: number of days to update. Cannot be set too large a date.
Default 10 days.
"""
w = YComDataExtr()
w.set_full_stocklist_to_retrieve(stock_sym_list)
w.set_hist_data_num_day_fr_current(num_days_for_updates)
w.get_all_hist_data()
## save to one particular funciton
# save to sql -- hist table
w.datatype_com_data_allstock_df.to_sql(
self.hist_data_tablename,
self.con,
flavor="sqlite",
schema=None,
if_exists="append",
index=True,
index_label=None,
chunksize=None,
dtype=None,
)
print "Getting Trends data -- Done. \n"
full_stock_data_df.to_csv(final_store_filename, index = False)
if 'd_pre' in partial_run:
""" Skip the company data info and just join based on store data."""
print 'Use backup data for company data.'
store_com_path = r'C:\data\stock_sql_db\company_data.csv'
com_data_df = pandas.read_csv(store_com_path)
full_stock_data_df = pandas.merge(full_stock_data_df, com_data_df, on = 'SYMBOL', how ='left')
if 'd' in partial_run:
## Replaced with the com data scraping using YQL --> some problem for this
print "Getting company data from YF using YQL"
print
print 'len of stock dataframe', len(full_stock_data_df)
dd = YComDataExtr()
dd.set_stock_sym_append_str('')
dd.set_full_stocklist_to_retrieve(list(full_stock_data_df['SYMBOL']))
dd.retrieve_all_results()
## will have to merge the two data set
full_stock_data_df = pandas.merge(full_stock_data_df, dd.com_data_allstock_df, on = 'SYMBOL', how ='left')
## save the company data so next time only need to read back and append
store_com_path = r'C:\data\stock_sql_db\company_data.csv'
dd.com_data_allstock_df.to_csv(store_com_path,index = False)
if 'e' in partial_run:
## tech analysis
print 'Tech analysis '
sym_list = list(full_stock_data_df['SYMBOL'])
rows = cur.fetchall() # return list of tuples
#do not have the header --> add in header and convert to pandas data
con.close()
if selection == 3:
""" Divident data might not need to collect so often"""
""" Deal with YQL and extract recent data """
""" Hist price retrieve using YQL. """
file = r'c:\data\full_Dec29.csv'
full_stock_data_df = pandas.read_csv(file)
w = YComDataExtr()
w.set_full_stocklist_to_retrieve(list(full_stock_data_df['SYMBOL'])[:100])
w.get_all_hist_data()
print w.datatype_com_data_allstock_df.head() # str away use the pandas to sql version to send??
## Users parameters
hist_data_tablename = 'histprice'
divdnt_data_tablename = 'dividend'
## initialized the database
con = lite.connect(db_full_path)
## save to one particular funciton
#save to sql -- hist table
w.datatype_com_data_allstock_df.to_sql(hist_data_tablename, con, flavor='sqlite',
schema=None, if_exists='replace', index=True,
cur.execute(command_str)
rows = cur.fetchall() # return list of tuples
# do not have the header --> add in header and convert to pandas data
con.close()
if selection == 3:
""" Divident data might not need to collect so often"""
""" Deal with YQL and extract recent data """
""" Hist price retrieve using YQL. """
file = r"c:\data\full_Dec29.csv"
full_stock_data_df = pandas.read_csv(file)
w = YComDataExtr()
w.set_full_stocklist_to_retrieve(list(full_stock_data_df["SYMBOL"])[:100])
w.get_all_hist_data()
print w.datatype_com_data_allstock_df.head() # str away use the pandas to sql version to send??
## Users parameters
hist_data_tablename = "histprice"
divdnt_data_tablename = "dividend"
## initialized the database
con = lite.connect(db_full_path)
## save to one particular funciton
# save to sql -- hist table
w.datatype_com_data_allstock_df.to_sql(
hist_data_tablename,
