def run(conf = {}):
"""
调度整个模块。
"""
date = conf.get('date',Date.getDate())
logging.info("Statistics module is started.Run date is %s" % date)
conf["projectPath"] = os.path.join(conf.get("SYS_HOME"),conf.get("STATISTICS_PROJECT_PATH"))
check = Check.Check(conf = conf)
if not check.checkStatisticsCondition(date = date):
logging.warning("Statistics is invailable.date:%s" % date)
return False
accuracy=Accuracy(conf)
accuracyResult = accuracy.compute(date = date)
if accuracyResult !=False:
path = os.path.join(conf.get("SYS_HOME",SYS_HOME),conf.get("ACCURACY_DATA_PATH",ACCURACY_DATA_PATH),date+SUFFIX)
Dumper.dump(data = accuracyResult,path = path)
logging.info("Accuracy statistics run success.Result is dump to %s" % path )
logging.debug("Accuracy statistics run result %s" % accuracyResult)
else:
logging.info("Accuracy statistics run failed.")
profile = Profile(conf)
profileResult = profile.profile()
if profileResult !=False:
path = os.path.join(conf.get("SYS_HOME",SYS_HOME),conf.get("PROFILE_PATH",PROFILE_DATA_PATH),date+SUFFIX)
Dumper.dump(data = profileResult,path = path)
logging.info("Profile statistics run success.Result is dump to %s"%path)
logging.debug("Profile statistics run result %s" % profileResult)
return True
else:
logging.info("Profile statistics run failed.")
return False
def run(conf={}):
stock=conf.get("stock",Stock(conf))
date=conf.get('date',Date.getDate())
logging.info("Start strategic app.Date is %s" %date)
check = Check.Check(conf = conf)
if not check.checkStrategicCondition(date = date):
logging.warning("Strategic is invailable.date:%s" % date)
return False
strategic_conf ={}
strategic_conf.update(conf)
strategic_conf['stock']=stock
strategic = Strategic(conf = strategic_conf)
strategicResult = strategic.compute()
path = os.path.join(conf.get("SYS_HOME"),conf.get("STRATEGIC_DATA_PATH",STRATEGIC_DATA_PATH),date+SUFFIX)
logging.debug("Dump strategic result to %s.\nStrategic result is:\n %s" % (strategic.data,path))
Dumper.dump(path = path,data = strategicResult)
logging.info("Finish strategic compute of date %s "%date)
logging.info("Start Bayes statistics.")
bayesHandler = Bayes(conf)
bayes = bayesHandler.posterior()
if bayes != False:
Dumper.dump(data = bayes,\
path = os.path.join(conf.get("SYS_HOME"),conf.get("BAYES_DATA_PATH",BAYES_DATA_PATH),date+SUFFIX))
logging.info("Bayes statistics run success.")
logging.debug("Bayes statistics run result %s" % bayes)
return True
else:
logging.info("Bayes statistics run fail.")
return False
def run(conf={}):
logging.info("Start compute app.")
logging.debug("Compute init conf is %s" % str(conf))
date=conf.get('date',Date.getDate())
stock=conf.get("stock",Stock(conf))
extract = Extract(conf = conf)
extract.extract(project = "fromDetailToAdv", stock = stock, date = date)
stock.data.dump()
return True
def run(conf={}):
"""
@param {Dict} conf
@param conf.date 抓取日期
@param {'min'|'detail'|'info'} [conf.download] 数据更新内容.day为日线数据，detail为详细数据,info为股票代码,对应的公司名等数据
@param {'detail'|'all'|'day'} [conf.mode] 当数据抓取模式为'day'时,'day'数据抓取模式。detail为从detail数据中抓取数据。all为抓取所有数据。day为更新每日数据
@param {'single'|'mutil'} [conf.thread] 当数据抓取模式为'detail'时,采取的线程模式。single为单线程,mutil为多线程
"""
logging.info("Start spider.")
downloader=Downloader(conf)
check = Check(conf)
date = conf.get("date",Date.getDate())
checkIndex = 0
while checkIndex < conf.get("MIN_DATA_NUM",MIN_DATA_NUM):
checkDate=Date.getDate(0-checkIndex,date)
checkReport = check.check(checkDate)
if not checkReport:
logging.info("Download %s 's data." % checkDate)
downloader.download(name="detail",conf={"date":checkDate})
checkIndex += 1
logging.info("Download finish.")
return True
def run(conf = {}):
"""
传入日期，读入projects和传入日期的statistics.进行计算，得到符合条件的strategic和股票code
例如一个report的project为“短线股票池”,里面定义了多少天内上涨概率大于多少的公式。如果某策略的统计数据符合该公式，则返回该策略的股票代码及其他相关信息。
>> reporter.run({"statistics":case['statistics']})
更新 statistics => probability
注:这里statistics主要指统计得到的bayes值.
"""
logging.info("Report program start.")
logging.debug("Report config %s"%str(conf))
conf['REPORTER_PROJECT_PATH']=os.path.join(\
conf.get("SYS_HOME",""),\
conf.get("REPORTER_PROJECT_PATH",REPORTER_PROJECT_PATH))
options={}
options.update(conf)
options.update(conf)
conf=options
date=conf.get('date',Date.getDate())
check = Check.Check(conf = conf)
if not check.checkReporterCondition(date = date):
logging.warning("Reporter is invailable.date:%s" % date)
return False
#get statistics
path = os.path.join(conf.get("SYS_HOME"),\
conf.get("BAYES_DATA_PATH",BAYES_DATA_PATH),\
date+SUFFIX)
statistics = conf.get("statistics" , Reader.read(path = path)) #载入指定日期的监控策略
if not statistics:
logging.warning("Reporter not find statistics data.Exit it.")
return None
#filter by conditions
strategic = Strategic(date = date ,conf = conf)
report = Report(conf = conf)
report.compute(strategic,statistics)
#dump result
if report != False:
path = os.path.join(conf.get("SYS_HOME",""),\
conf.get("REPORTER_DATA_PATH",REPORTER_DATA_PATH),date+SUFFIX)
Dumper.dump(path = path , data = report.data)
logging.info("Reporter result : %s" % str(report.data))
else:
logging.info("Reporter run fail.")
return report
def run(conf={}):
date = Date.getInstantDate() #instant date 应该Date.getDate +1,key的编码问题
conf['date']=date
logging.info("start downloading instant data.Date is %s" % conf['date'])
downloader = Downloader(conf = conf)
downloader.download("instant")
logging.info("start computing instant data.Date is %s"% conf['date'])
stock=Stock(conf)
extract = Extract(conf = conf)
extract.extract(project = "instant", stock = stock, date = date)
logging.info("start instant strategic data.Date is %s" %conf['date'])
strategic_conf ={}
strategic_conf.update(conf)
strategic_conf['stock']=stock
strategic = Strategic(conf = strategic_conf)
strategicResult = strategic.compute()
Dumper.dump(path = os.path.join(conf.get("SYS_HOME"),conf.get("INSTANT_DATA_PATH"),"strategic.json"),data = strategicResult)
logging.info("Start Bayes statistics.Date is %s"% conf['date'])
conf['strategic']=strategic
bayesHandler = Bayes(conf)
bayes = bayesHandler.posterior()
Dumper.dump(path = os.path.join(conf.get("SYS_HOME"),conf.get("INSTANT_DATA_PATH"),"bayes.json"),data = bayes)
logging.info("Start report.Date is %s"% conf['date'])
report = Report(conf = conf)
report.compute(strategic = strategic , statistics = bayes)
print report
#profile
profile_conf = {"stock":stock,"date":date}
profile_conf.update(conf)
profile = Profile(profile_conf)
profile.viewProfile()
path = os.path.join(conf.get("SYS_HOME"),conf.get("INSTANT_REPORT_DATA_PATH"))
Dumper.dump(path = path,data = report.data)
return report.data
def run(conf={}):
"""
传入日期，读入projects和传入日期的statistics.进行计算，得到符合条件的strategic和股票code
例如一个report的project为“短线股票池”,里面定义了多少天内上涨概率大于多少的公式。如果某策略的统计数据符合该公式，则返回该策略的股票代码及其他相关信息。
>> reporter.run({"statistics":case['statistics']})
更新 statistics => probability
注:这里statistics主要指统计得到的bayes值.
"""
logging.info("Report program start.")
logging.debug("Report config %s" % str(conf))
if not "stock" in conf:
conf["stock"] = Stock(conf)
if not "date" in conf:
conf["date"] = Date.getDate()
# load train/statistics data
if "statistics" in conf:
statistics = conf.get("statistics") # 载入指定日期的监控策略
else:
path = os.path.join(conf.get("SYS_HOME"), conf.get("BAYES_DATA_PATH", BAYES_DATA_PATH), conf["date"] + SUFFIX)
statistics = Reader.read(path=path)
# predict
logging.info("Start strategic app.Date is %s" % conf["date"])
strategic = Strategic(conf=conf)
strategic.check()
strategic.compute()
strategic.dump()
logging.info("Finish strategic compute of date %s " % conf["date"])
# generate report
report = Report(conf=conf)
report.compute(strategic, statistics)
report.check()
report.dump()
return report
logging.error("No %s commands." % cmd_name)
else:
cmd = imp.load_module(cmd_name, fp, pathname, description)
cmd.run(conf = conf)
finally:
if fp:
fp.close()
if __name__=="__main__" :
if sys.argv[1:]:
conf=Conf.load(\
os.path.join(SYS_HOME,"conf","stock.yaml"),
os.path.join(SYS_HOME,"conf","downloader.yaml"))
conf["SYS_HOME"]=SYS_HOME
conf.update(parseArgs(conf))
Log.set(os.path.join(SYS_HOME,conf.get("LOG_PATH")))
action=conf.get("action","run")
dates=conf.get("dates") #如果指定多个日期，则按顺序启动实例一个一个运行
stockCache = {} #@todo add stockCache
if not dates or len(dates)==0:
conf['date']=Date.getDate()
execute(cmd_name = action,conf=conf)
elif len(dates)>=1:
for sDate in dates:
conf["date"]=sDate
if stockCache:
conf['stock']=stockCache
execute(cmd_name = action,conf=conf)
else:
print __doc__
