def getstats():
df_all = pd.DataFrame()
#list_symbol=dbdao.get_symbols_list_limit(start,end)
list_symbol = dbdao.get_missing_stats_symbol(start, end)
if (start == '0'):
dbdao.execute_query(["delete from df_stats"])
print list_symbol
logger.info(list_symbol)
for symbol in list_symbol:
try:
systats_data = stats_manager.yf_get_key_stat(symbol)
logger.info('calculating for symbol ' + symbol)
systats_data.append(['symbol', symbol])
df = pd.DataFrame(systats_data)
if (df.shape[0] > 10):
df = df.rename(columns={0: 'name'})
df['name'] = df['name'].apply(correct_name)
df = df[df['name'] != 'drop']
df = df.set_index('name').transpose()
print df
df_all = df_all.append(df)
except Exception, ex:
logger.error(ex)
def getstats():
df_all= pd.DataFrame()
#list_symbol=dbdao.get_symbols_list_limit(start,end)
list_symbol=dbdao.get_missing_stats_symbol(start,end)
if(start=='0'):
dbdao.execute_query(["delete from df_stats"])
print list_symbol
logger.info(list_symbol)
for symbol in list_symbol:
try:
systats_data = stats_manager.yf_get_key_stat(symbol)
logger.info('calculating for symbol '+symbol)
systats_data.append(['symbol',symbol])
df= pd.DataFrame(systats_data)
if(df.shape[0] >10):
df = df.rename(columns={0: 'name'})
df['name']= df['name'].apply(correct_name)
df= df[df['name']!='drop']
df= df.set_index('name').transpose()
print df
df_all=df_all.append(df)
except Exception,ex:
logger.error(ex)
def calculate_technicals(start,end):
end_date_time = datetime.datetime.now() # mysqldb.get_maxdate()[0]
start_date_time = end_date_time - relativedelta(days=constants.DAYS_FOR_TECHNICALS)
list_symbol=dbdao.get_symbols_list_limit(start,end)
list_symbol=['MSFT']
hist_dates= dbdao.get_historical_dates()
days_behind=100
df_mkt=mongodao.getsymbol_data(constants.MKT_SYMBOL, start_date_time, end_date_time)
if(start=='0'):
dbdao.execute_query(["drop table df_technical","truncate table df_history","truncate table df_alerts"])
df_technicals = technical_manager.calculate_technical(df_mkt,constants.MKT_SYMBOL,df_mkt, start_date_time, end_date_time, hist_dates,days_behind)
frames=[df_technicals]
for symbol in list_symbol:
try:
df_symbol=mongodao.getsymbol_data(symbol, start_date_time, end_date_time)
if df_symbol.empty:
continue
logger.info("Getting Technicals for symbol=%s ",symbol)
df_technicals_new = technical_manager.calculate_technical(df_symbol,symbol,df_mkt, start_date_time, end_date_time, hist_dates,days_behind)
frames.append(df_technicals_new)
logger.info("Got Technicals for symbol=%s ",symbol)
except Exception ,ex:
logger.error(ex)
logger.error( traceback.format_exc())
def getgoogledata(list_symbol):
symbols = str.join(',',list_symbol)
print symbols
url = 'http://finance.google.com/finance/info?q=%s' % symbols
print url
page = urllib2.urlopen(url,timeout = 10)
html = page.read().replace("//","").strip()
data = json.loads(html)
df=pd.DataFrame(data)
df.set_index('t',inplace=True)
return df
dbdao.execute_query(['delete from google_live_indices_symbol','delete from google_live_symbol'])
list_symbol=dbdao.get_indices_symbols_list()
print list_symbol
df=getgoogledata(list_symbol)
dbdao.save_dataframe(df, "google_live_indices_symbol")
list_symbol=dbdao.get_symbols_list()
for i in range(0, len(list_symbol), 40):
try:
chunk = list_symbol[i:i + 40]
df=getgoogledata(chunk)
dbdao.save_dataframe(df, "google_live_symbol")
logger.info( chunk)
except Exception,ex:
# logger.error(e)
# exit()
def getdataforall_list(list_symbols):
try:
for i in range(0, len(list_symbols), 40):
chunk = list_symbols[i:i + 40]
logger.info(chunk)
saveQuote(chunk)
except Exception, ex:
logger.error(ex)
dbdao.execute_query(["delete from yahoo_live_symbol"])
list_symbol = dbdao.get_symbols_list()
getdataforall_list(list_symbol)
update_sql = """ update live_symbol t1,yahoo_live_symbol t2
set
t1.52weekhigh=t2.52weekhigh,
t1.52weeklow=t2.52weeklow,
t1.PE=t2.PE,
t1.avg_daily_volume=t2.avg_daily_volume,
t1.book_value=t2.book_value,
t1.change_pct=t2.change_pct,
t1.dividend=t2.dividend,
t1.eps=t2.eps,
t1.earning_per_share=t2.earning_per_share,
t1.ex_dividend_date=t2.ex_dividend_date,
from dao import dbdao
from util import df_util
dbdao.execute_query(["delete from df_stats_quarter"])
df = df_util.create_dataframe("data\\stats\\balance_sheet", "Symbol", "Balance Sheet", "df_stats_quarter")
df = df_util.create_dataframe("data\\stats\\income_statement", "Symbol", "Income Statement", "df_stats_quarter")
df = df_util.create_dataframe("data\\stats\\cash_flow", "Symbol", "Cash Flow", "df_stats_quarter")
from dao import dbdao
from util import loglib
logger = loglib.getlogger('notable moves')
f=open('queries/notable_moves.sql')
queries=f.read().strip().split(';')
print len(queries)
#last query is empty
queries=queries[:len(queries)-1]
dbdao.execute_query(queries)
from dao import dbdao
from util import loglib
logger = loglib.getlogger('notable moves')
f = open('queries/notable_moves.sql')
queries = f.read().strip().split(';')
print len(queries)
#last query is empty
queries = queries[:len(queries) - 1]
dbdao.execute_query(queries)
# add extra columns for low high median
df_technical['low_5day_max']=0
df_technical['low_5day_min']=0
df_technical['low_5day_median']=0
df_technical['high_5day_max']=0
df_technical['high_5day_min']=0
df_technical['high_5day_median']=0
df_technical['rsi_text']=''
dbdao.save_dataframe(df_technical,"df_technical");
sql_max_min_median_5days = open('queries/high_low_median.sql', 'r').read()
sql_update_technical_history = open('queries/update_technical_history.txt', 'r').read()
logger.info("Executing Queries for updating technicals")
delete_technicals_latest="delete from technicals_symbol"
update_technical_latest = open('queries/update_technical_latest.txt', 'r').read()
dbdao.execute_query([sql_max_min_median_5days,sql_update_technical_history,delete_technicals_latest,update_technical_latest])
logger.info("Queries Completed !")
import sys
print sys.argv
start,end=sys.argv[1],sys.argv[2]
calculate_technicals(start,end)
