def main():
args = parse_args()
configFilePath = args.conf
trainingStart = args.tStart
trainingEnd = args.tEnd
estimationStart = args.eStart
estimationEnd = args.eEnd
key_num = args.key_numbers
clu_num = args.c_numbers
days_back = args.days_back
common.init(configFilePath)
execute(trainingStart,trainingEnd,estimationStart,estimationEnd,key_num,clu_num,days_back)
def main():
#get stock index values
cfgPath = "../Config/config.cfg"
common.init(cfgPath)
con = common.getDBConnection()
cur = con.cursor()
sql = "select stock_index,date,zscore30,zscore90,one_day_change/(last_price-one_day_change) from t_daily_stockindex where date<'2011-01-01' order by date asc"
cur.execute(sql)
rows = cur.fetchall()
stockBasket = {}
tmpStockBasket = {}
for row in rows:
stockIndex = row[0]
date = row[1]
zscore30 = row[2]
zscore90 = row[3]
lastPrice = row[4]
if stockIndex not in stockBasket:
stockBasket[stockIndex] = []
tmpStockBasket[stockIndex] = {}
dayValue = {}
dayValue["date"] = date
dayValue["zscore30"] = zscore30
dayValue["zscore90"] = zscore90
dayValue["lastPrice"] = lastPrice
stockBasket[stockIndex].append(dayValue)
tmpStockBasket[stockIndex][date] = [zscore30,zscore90,lastPrice]
con.close()
stockList = []
#compute the pearson for each two stock
for stockIndex in stockBasket:
stockList.append(stockIndex)
print stockList
for i in range(0,9):
for j in range(i+1,9):
print stockList[i],stockList[j]
stock1 = tmpStockBasket[stockList[i]]
stock2 = tmpStockBasket[stockList[j]]
compare_stock(stock1,stock2)
for i in range(0,9):
for j in range(0,9):
print stockList[i],stockList[j]
stock1 = stockBasket[stockList[i]]
stock2 = stockBasket[stockList[j]]
compare_stock_shift(stockList[i],stockList[j],stock1,stock2,0)
def testPhase(stock1,stock2,z30m,z30c,z90m,z90c):
cfgPath = "../Config/config.cfg"
common.init(cfgPath)
con = common.getDBConnection()
cur = con.cursor()
sql = "select stock_index,date,zscore30,zscore90,one_day_change/(last_price-one_day_change) from t_daily_stockindex where date>='2011-01-01' and stock_index=? order by date asc"
cur.execute(sql,(stock1,))
rows = cur.fetchall()
for row in rows:
date = row[1]
zscore30 = float(row[2])
zscore90 = float(row[3])
predictZ30 = z30m * zscore30 + z30c
predictZ90 = z90m * zscore90 + z90c
if abs(predictZ30) >= 4 or abs(predictZ90) >= 3:
print "{} : using {} to predict {} z30: {} z90: {}".format(date,stock1,stock2,predictZ30,predictZ90)
con.close()
news["content"] = article["content"]
news["stockIndex"] = stockIndex
news["source"] = "Bloomberg News"
news["updateTime"] = article["queryTime"]
news["url"] = article["newsUrl"]
embersId = hashlib.sha1(article["content"]).hexdigest()
news["embersId"] = embersId
ifExisted = check_article_existed(news)
if not ifExisted:
insert_news(news)
"Insert into Mission process"
insert_news_mission(news)
i = i + 1
if i % 1000 == 0:
con.commit()
con.commit()
except lite.Error, e:
print "Error: %s" % e.args[0]
finally:
con.close()
def parse_args():
ap = argparse.ArgumentParser("Import the archived news")
ap.add_argument('-c','--conf',metavar='CONFIG',type=str,default='../Config/config.cfg',nargs='?',help='The path of config file')
return ap.parse_args()
if __name__ == "__main__":
args = parse_args()
conf = args.conf
common.init(conf)
import_news_to_database()
def __init__(self,cfgPath):
common.init(cfgPath)
logs.init()
def init(cfgPath):
common.init(cfgPath)
print "RawNewsProcess End Time: ",datetime.strftime(datetime.now(),"%Y-%m-%d %H:%M:%S")
# "Import Enriched Data Start"
# print "Import Enriched Data Start Time: ",datetime.strftime(datetime.now(),"%Y-%m-%d %H:%M:%S")
# OutputTestStockIndexData.export_test_stock_data(estimationStart,estimationEnd)
# print "Import Enriched Data End Time: ",datetime.strftime(datetime.now(),"%Y-%m-%d %H:%M:%S")
def parse_args():
ap = argparse.ArgumentParser("The automatic training and test tools")
ap.add_argument('-c','--conf',metavar='CONFIG',type=str,default='../Config/config.cfg',nargs='?',help='The path of config file')
ap.add_argument('-ts','--tStart',metavar='TRAING START',type=str,nargs='?',help='The start day of Training set')
ap.add_argument('-td','--tEnd',metavar='TRAING END',type=str,nargs='?',help='The end day of Training set')
ap.add_argument('-es','--eStart',metavar='ESTIMATION START',type=str,nargs='?',help='The start day of estimation set')
ap.add_argument('-ed','--eEnd',metavar='ESTIMATION End',type=str,nargs='?',help='The end day of estimation set')
return ap.parse_args()
if __name__ == "__main__":
args = parse_args()
configFilePath = args.conf
trainingStart = args.tStart
trainingEnd = args.tEnd
estimationStart = args.eStart
estimationEnd = args.eEnd
common.init(configFilePath)
execute(trainingStart,trainingEnd,estimationStart,estimationEnd)
import xlrd
import sqlite3 as lite
from datetime import datetime
from Util import common
import json
cfgPath = "../Config/config.cfg"
common.init(cfgPath)
# read excel data and insert database
class ImportExcelData:
def read_data_from_excel( self ):
excelData = xlrd.open_workbook( "D:/Embers/GSR August 2012 V2.xls" )
sheetData = excelData.sheet_by_index( 0 )
numberOfRows = sheetData.nrows
dataList = []
# iterate each row to get the cell value and composite the data oject before inserting database
for rowNo in range( 1, numberOfRows ):
eventCode = sheetData.cell( rowNo, 7 ).value
if eventCode in ['0411', '0412', '0421', '0422']:
dataObject = {}
dataObject["country"] = sheetData.cell( rowNo, 4 ).value
dataObject["eventCode"] = sheetData.cell( rowNo, 7 ).value
dataObject["population"] = sheetData.cell( rowNo, 8 ).value
cellObject = sheetData.cell( rowNo, 9 )
if cellObject.ctype == xlrd.XL_CELL_DATE:
dateTuple = xlrd.xldate_as_tuple( cellObject.value, excelData.datemode )
def init(cfgPath):
common.init(cfgPath)
logs.init()
