def getstats(): df_all = pd.DataFrame() #list_symbol=dbdao.get_symbols_list_limit(start,end) list_symbol = dbdao.get_missing_stats_symbol(start, end) if (start == '0'): dbdao.execute_query(["delete from df_stats"]) print list_symbol logger.info(list_symbol) for symbol in list_symbol: try: systats_data = stats_manager.yf_get_key_stat(symbol) logger.info('calculating for symbol ' + symbol) systats_data.append(['symbol', symbol]) df = pd.DataFrame(systats_data) if (df.shape[0] > 10): df = df.rename(columns={0: 'name'}) df['name'] = df['name'].apply(correct_name) df = df[df['name'] != 'drop'] df = df.set_index('name').transpose() print df df_all = df_all.append(df) except Exception, ex: logger.error(ex)
def getstats(): df_all= pd.DataFrame() #list_symbol=dbdao.get_symbols_list_limit(start,end) list_symbol=dbdao.get_missing_stats_symbol(start,end) if(start=='0'): dbdao.execute_query(["delete from df_stats"]) print list_symbol logger.info(list_symbol) for symbol in list_symbol: try: systats_data = stats_manager.yf_get_key_stat(symbol) logger.info('calculating for symbol '+symbol) systats_data.append(['symbol',symbol]) df= pd.DataFrame(systats_data) if(df.shape[0] >10): df = df.rename(columns={0: 'name'}) df['name']= df['name'].apply(correct_name) df= df[df['name']!='drop'] df= df.set_index('name').transpose() print df df_all=df_all.append(df) except Exception,ex: logger.error(ex)
def calculate_technicals(start,end): end_date_time = datetime.datetime.now() # mysqldb.get_maxdate()[0] start_date_time = end_date_time - relativedelta(days=constants.DAYS_FOR_TECHNICALS) list_symbol=dbdao.get_symbols_list_limit(start,end) list_symbol=['MSFT'] hist_dates= dbdao.get_historical_dates() days_behind=100 df_mkt=mongodao.getsymbol_data(constants.MKT_SYMBOL, start_date_time, end_date_time) if(start=='0'): dbdao.execute_query(["drop table df_technical","truncate table df_history","truncate table df_alerts"]) df_technicals = technical_manager.calculate_technical(df_mkt,constants.MKT_SYMBOL,df_mkt, start_date_time, end_date_time, hist_dates,days_behind) frames=[df_technicals] for symbol in list_symbol: try: df_symbol=mongodao.getsymbol_data(symbol, start_date_time, end_date_time) if df_symbol.empty: continue logger.info("Getting Technicals for symbol=%s ",symbol) df_technicals_new = technical_manager.calculate_technical(df_symbol,symbol,df_mkt, start_date_time, end_date_time, hist_dates,days_behind) frames.append(df_technicals_new) logger.info("Got Technicals for symbol=%s ",symbol) except Exception ,ex: logger.error(ex) logger.error( traceback.format_exc())
def getgoogledata(list_symbol): symbols = str.join(',',list_symbol) print symbols url = 'http://finance.google.com/finance/info?q=%s' % symbols print url page = urllib2.urlopen(url,timeout = 10) html = page.read().replace("//","").strip() data = json.loads(html) df=pd.DataFrame(data) df.set_index('t',inplace=True) return df dbdao.execute_query(['delete from google_live_indices_symbol','delete from google_live_symbol']) list_symbol=dbdao.get_indices_symbols_list() print list_symbol df=getgoogledata(list_symbol) dbdao.save_dataframe(df, "google_live_indices_symbol") list_symbol=dbdao.get_symbols_list() for i in range(0, len(list_symbol), 40): try: chunk = list_symbol[i:i + 40] df=getgoogledata(chunk) dbdao.save_dataframe(df, "google_live_symbol") logger.info( chunk) except Exception,ex:
# logger.error(e) # exit() def getdataforall_list(list_symbols): try: for i in range(0, len(list_symbols), 40): chunk = list_symbols[i:i + 40] logger.info(chunk) saveQuote(chunk) except Exception, ex: logger.error(ex) dbdao.execute_query(["delete from yahoo_live_symbol"]) list_symbol = dbdao.get_symbols_list() getdataforall_list(list_symbol) update_sql = """ update live_symbol t1,yahoo_live_symbol t2 set t1.52weekhigh=t2.52weekhigh, t1.52weeklow=t2.52weeklow, t1.PE=t2.PE, t1.avg_daily_volume=t2.avg_daily_volume, t1.book_value=t2.book_value, t1.change_pct=t2.change_pct, t1.dividend=t2.dividend, t1.eps=t2.eps, t1.earning_per_share=t2.earning_per_share, t1.ex_dividend_date=t2.ex_dividend_date,
from dao import dbdao from util import df_util dbdao.execute_query(["delete from df_stats_quarter"]) df = df_util.create_dataframe("data\\stats\\balance_sheet", "Symbol", "Balance Sheet", "df_stats_quarter") df = df_util.create_dataframe("data\\stats\\income_statement", "Symbol", "Income Statement", "df_stats_quarter") df = df_util.create_dataframe("data\\stats\\cash_flow", "Symbol", "Cash Flow", "df_stats_quarter")
from dao import dbdao from util import loglib logger = loglib.getlogger('notable moves') f=open('queries/notable_moves.sql') queries=f.read().strip().split(';') print len(queries) #last query is empty queries=queries[:len(queries)-1] dbdao.execute_query(queries)
from dao import dbdao from util import loglib logger = loglib.getlogger('notable moves') f = open('queries/notable_moves.sql') queries = f.read().strip().split(';') print len(queries) #last query is empty queries = queries[:len(queries) - 1] dbdao.execute_query(queries)
# add extra columns for low high median df_technical['low_5day_max']=0 df_technical['low_5day_min']=0 df_technical['low_5day_median']=0 df_technical['high_5day_max']=0 df_technical['high_5day_min']=0 df_technical['high_5day_median']=0 df_technical['rsi_text']='' dbdao.save_dataframe(df_technical,"df_technical"); sql_max_min_median_5days = open('queries/high_low_median.sql', 'r').read() sql_update_technical_history = open('queries/update_technical_history.txt', 'r').read() logger.info("Executing Queries for updating technicals") delete_technicals_latest="delete from technicals_symbol" update_technical_latest = open('queries/update_technical_latest.txt', 'r').read() dbdao.execute_query([sql_max_min_median_5days,sql_update_technical_history,delete_technicals_latest,update_technical_latest]) logger.info("Queries Completed !") import sys print sys.argv start,end=sys.argv[1],sys.argv[2] calculate_technicals(start,end)