def calculate_history_dates():
#mysqldao=dbutil.mysqldb()
#print technicals.gethistory_dates()
header=("DateType,Date")
hist_dates=historical_dates_manager.gethistory_dates()
filename='data/historical_date/hist_dates'
f1=open(filename, 'wb')
f1.write(header)
f1.write('\n')
writer = csv.writer(f1)
for key, value in hist_dates.items():
writer.writerow([key, value])
f1.close()
delete_sql="delete from historicaldates"
insert_sql="""
LOAD DATA LOCAL INFILE '%s'
INTO TABLE historicaldates
FIELDS TERMINATED BY ','
LINES TERMINATED BY "\n"
IGNORE 1 LINES
(@DateType,@Date)
SET DateType=@DateType,Date=@Date
"""%(filename)
logger.info("Saving Historical Dates to database")
dbdao.execute_query([delete_sql,insert_sql])
logger.info("Saved Historical Dates to database")
def calculate_history_dates():
#mysqldao=dbutil.mysqldb()
#print technicals.gethistory_dates()
header = ("DateType,Date")
hist_dates = historical_dates_manager.gethistory_dates()
filename = 'data/historical_date/hist_dates'
f1 = open(filename, 'wb')
f1.write(header)
f1.write('\n')
writer = csv.writer(f1)
for key, value in hist_dates.items():
writer.writerow([key, value])
f1.close()
delete_sql = "delete from historicaldates"
insert_sql = """
LOAD DATA LOCAL INFILE '%s'
INTO TABLE historicaldates
FIELDS TERMINATED BY ','
LINES TERMINATED BY "\n"
IGNORE 1 LINES
(@DateType,@Date)
SET DateType=@DateType,Date=@Date
""" % (filename)
logger.info("Saving Historical Dates to database")
dbdao.execute_query([delete_sql, insert_sql])
logger.info("Saved Historical Dates to database")
try:
logger.info("Getting news for" + url_name)
for entry in feed["entries"]:
list_entries=[]
list_entries.append(url_name)
list_entries.append(entry.title)
list_entries.append(entry.summary)
list_entries.append(entry.link)
list_entries.append(entry.published)
content=seperator.join(list_entries)
#print content
f1.write(content.encode('ascii', 'ignore').strip().replace("\n",""))
f1.write("\n")
except Exception ,ex:
logger.error(ex)
f1.close()
sql1="delete from news_details"
sql2="""LOAD DATA LOCAL INFILE '%s'
INTO TABLE news_details
FIELDS TERMINATED BY '%s'
LINES TERMINATED BY "\n"
IGNORE 1 LINES
(@col1,@col2,@col3,@col4,@col5) set source=@col1,title=@col2,description=@col3,link=@col4,pubDate=@col5
;"""%(filepath,seperator)
dbdao.execute_query([sql1,sql2])
for symbol in list_symbol:
try:
list_symbol_data.append(getsymbol_sector_industry(symbol))
except Exception,ex:
logger.error(ex)
df=pd.DataFrame(list_symbol_data)
df.set_index("symbol",inplace=True)
dbdao.save_dataframe(df, "df_symbol_sector_industry")
print df
dbdao.execute_query(["delete from df_symbol_sector_industry"])
list_symbol = dbdao.get_symbols_list_missing()
create_sector_industry_data(list_symbol)
sql_insert=""" insert into list_symbol(symbol,companyname,sectorid,industryid)
select t1.symbol,t1.name,s1.id,i1.industryid from df_symbol_sector_industry t1
left join sectors s1 on t1.sector=s1.name
left join industries i1 on t1.industry=i1.industryname"""
dbdao.execute_query([sql_insert])
# import sys
# print sys.argv
# start,end=sys.argv[1],sys.argv[2]
# print start,end
#print getsymbol_sector_industry('XLV')